{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Read Data Sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-14T18:48:56.875761Z",
     "start_time": "2017-06-14T18:48:56.409705Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "import time\n",
    "from collections import namedtuple\n",
    "pd.set_option(\"display.max_rows\",35)\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-14T18:48:56.999801Z",
     "start_time": "2017-06-14T18:48:56.877402Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class dataset:\n",
    "    kdd_train_2labels = pd.read_pickle(\"dataset/kdd_train_2labels.pkl\")\n",
    "    kdd_test_2labels = pd.read_pickle(\"dataset/kdd_test_2labels.pkl\")\n",
    "    kdd_test__2labels = pd.read_pickle(\"dataset/kdd_test__2labels.pkl\")\n",
    "    \n",
    "    kdd_train_5labels = pd.read_pickle(\"dataset/kdd_train_5labels.pkl\")\n",
    "    kdd_test_5labels = pd.read_pickle(\"dataset/kdd_test_5labels.pkl\")\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-14T18:48:57.007302Z",
     "start_time": "2017-06-14T18:48:57.001507Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(125973, 124)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.kdd_train_2labels.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-14T18:48:57.030267Z",
     "start_time": "2017-06-14T18:48:57.009259Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(22544, 124)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.kdd_test_2labels.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-14T18:48:57.874293Z",
     "start_time": "2017-06-14T18:48:57.031996Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.99589320646770185"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn import model_selection as ms\n",
    "from sklearn import preprocessing as pp\n",
    "\n",
    "class preprocess:\n",
    "    \n",
    "    output_columns_2labels = ['is_Normal','is_Attack']\n",
    "    \n",
    "    x_input = dataset.kdd_train_2labels.drop(output_columns_2labels, axis = 1)\n",
    "    y_output = dataset.kdd_train_2labels.loc[:,output_columns_2labels]\n",
    "\n",
    "    x_test_input = dataset.kdd_test_2labels.drop(output_columns_2labels, axis = 1)\n",
    "    y_test = dataset.kdd_test_2labels.loc[:,output_columns_2labels]\n",
    "    \n",
    "    x_test__input = dataset.kdd_test__2labels.drop(output_columns_2labels, axis = 1)\n",
    "    y_test_ = dataset.kdd_test__2labels.loc[:,output_columns_2labels]\n",
    "\n",
    "    ss = pp.StandardScaler()\n",
    "\n",
    "    x_train = ss.fit_transform(x_input)\n",
    "    x_test = ss.transform(x_test_input)\n",
    "    x_test_ = ss.transform(x_test__input)\n",
    "\n",
    "    y_train = y_output.values\n",
    "    y_test = y_test.values\n",
    "    y_test_ = y_test_.values\n",
    "\n",
    "    \n",
    "preprocess.x_train.std()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-14T18:48:59.776591Z",
     "start_time": "2017-06-14T18:48:57.876167Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.contrib.legacy_seq2seq.python.ops.seq2seq import basic_rnn_seq2seq\n",
    "from tensorflow.contrib.rnn import RNNCell, LSTMCell, MultiRNNCell\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-14T18:49:00.044268Z",
     "start_time": "2017-06-14T18:48:59.778361Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class network(object):\n",
    "    \n",
    "    input_dim = 122\n",
    "    classes = 2\n",
    "    hidden_encoder_dim = 122\n",
    "    hidden_layers = 1\n",
    "    latent_dim = 10\n",
    "\n",
    "    hidden_decoder_dim = 122\n",
    "    lam = 0.01\n",
    "    \n",
    "    def __init__(self, classes, hidden_layers, num_of_features):\n",
    "        self.classes = classes\n",
    "        self.hidden_layers = hidden_layers\n",
    "        self.latent_dim = num_of_features\n",
    "            \n",
    "    def build_layers(self):\n",
    "        tf.reset_default_graph()\n",
    "        #learning_rate = tf.Variable(initial_value=0.001)\n",
    "\n",
    "        input_dim = self.input_dim\n",
    "        classes = self.classes\n",
    "        hidden_encoder_dim = self.hidden_encoder_dim\n",
    "        hidden_layers = self.hidden_layers\n",
    "        latent_dim = self.latent_dim\n",
    "        hidden_decoder_dim = self.hidden_decoder_dim\n",
    "        lam = self.lam\n",
    "        \n",
    "        with tf.variable_scope(\"Input\"):\n",
    "            self.x_input = tf.placeholder(\"float\", shape=[None, 1, input_dim])\n",
    "            self.y_input_ = tf.placeholder(\"float\", shape=[None, 1, classes])\n",
    "            self.keep_prob = tf.placeholder(\"float\")\n",
    "            self.lr = tf.placeholder(\"float\")\n",
    "            self.x_list = tf.unstack(self.x_input, axis= 1)\n",
    "            self.y_list_ = tf.unstack(self.y_input_, axis = 1)\n",
    "            self.y_ = self.y_list_[0]\n",
    "            \n",
    "            #GO = tf.fill((tf.shape(self.x)[0], 1), 0.5)\n",
    "            \n",
    "            #y_with_GO = tf.stack([self.y_, GO])\n",
    "            \n",
    "        with tf.variable_scope(\"lstm\"):\n",
    "            multi_cell = MultiRNNCell([LSTMCell(input_dim) for i in range(hidden_layers)] )\n",
    "            \n",
    "            self.y, states = basic_rnn_seq2seq(self.x_list, self.y_list_, multi_cell)\n",
    "            #self.y = tf.slice(self.y, [0, 0], [-1,2])\n",
    "            \n",
    "            #self.out = tf.squeeze(self.y)\n",
    "            \n",
    "            self.y = tf.layers.dense(self.y[0], classes, activation = tf.nn.softmax)\n",
    "        with tf.variable_scope(\"Loss\"):\n",
    "            \n",
    "            self.regularized_loss = tf.losses.mean_squared_error(self.y_, self.y)\n",
    "            correct_prediction = tf.equal(tf.argmax(self.y_, 1), tf.argmax(self.y, 1))\n",
    "            self.tf_accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32), name = \"Accuracy\")\n",
    "\n",
    "        with tf.variable_scope(\"Optimizer\"):\n",
    "            learning_rate=self.lr\n",
    "            optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "            gradients, variables = zip(*optimizer.compute_gradients(self.regularized_loss))\n",
    "            gradients = [\n",
    "                None if gradient is None else tf.clip_by_value(gradient, -1, 1)\n",
    "                for gradient in gradients]\n",
    "            self.train_op = optimizer.apply_gradients(zip(gradients, variables))\n",
    "            #self.train_op = optimizer.minimize(self.regularized_loss)\n",
    "            \n",
    "        # add op for merging summary\n",
    "        #self.summary_op = tf.summary.merge_all()\n",
    "        self.pred = tf.argmax(self.y, axis = 1)\n",
    "        self.actual = tf.argmax(self.y_, axis = 1)\n",
    "\n",
    "        # add Saver ops\n",
    "        self.saver = tf.train.Saver()\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-01T00:59:00.684124Z",
     "start_time": "2017-06-01T00:58:59.843181Z"
    }
   },
   "source": [
    "batch_iterations = 200\n",
    "\n",
    "x_train, x_valid, y_train, y_valid, = ms.train_test_split(preprocess.x_train, \n",
    "                                                                          preprocess.y_train, \n",
    "                                                                          test_size=0.1)\n",
    "batch_indices = np.array_split(np.arange(x_train.shape[0]), \n",
    "                                           batch_iterations)\n",
    "                                                                          \n",
    "for i in batch_indices:\n",
    "    print(x_train[i,np.newaxis,:])\n",
    "    print(y_train[i,np.newaxis,:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-14T18:49:00.342052Z",
     "start_time": "2017-06-14T18:49:00.045902Z"
    },
    "collapsed": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import collections\n",
    "\n",
    "class Train:    \n",
    "    \n",
    "    result = namedtuple(\"score\", ['epoch', 'no_of_features','hidden_layers','train_score', 'test_score', 'test_score_20', 'time_taken'])\n",
    "\n",
    "    predictions = {}\n",
    "\n",
    "    results = []\n",
    "    best_acc = 0\n",
    "    best_acc_global = 0\n",
    "\n",
    "    def train(epochs, net, h,f, lrs):\n",
    "        batch_iterations = 200\n",
    "        train_loss = None\n",
    "        Train.best_acc = 0\n",
    "        os.makedirs(\"dataset/tf_lstm_nsl_kdd/hidden layers_{}_features count_{}\".format(h,f),\n",
    "                    exist_ok = True)\n",
    "        with tf.Session() as sess:\n",
    "            #summary_writer_train = tf.summary.FileWriter('./logs/kdd/VAE/training', graph=sess.graph)\n",
    "            #summary_writer_valid = tf.summary.FileWriter('./logs/kdd/VAE/validation')\n",
    "\n",
    "            sess.run(tf.global_variables_initializer())\n",
    "            start_time = time.perf_counter()\n",
    "            \n",
    "            accuracy, pred_value, actual_value, y_pred = sess.run([net.tf_accuracy, \n",
    "                                                                   net.pred, \n",
    "                                                                   net.actual, net.y], \n",
    "                                                                  feed_dict={net.x_input: preprocess.x_test[:,np.newaxis,:], \n",
    "                                                                             net.y_input_: preprocess.y_test[:,np.newaxis,:], \n",
    "                                                                             net.keep_prob:1})\n",
    "            \n",
    "            print(\"Initial Accuracy, before training: {}\".format(accuracy))\n",
    "            \n",
    "            for lr in lrs:\n",
    "                for epoch in range(1, (epochs+1)):\n",
    "                    x_train, x_valid, y_train, y_valid, = ms.train_test_split(preprocess.x_train, \n",
    "                                                                              preprocess.y_train, \n",
    "                                                                              test_size=0.1)\n",
    "                    batch_indices = np.array_split(np.arange(x_train.shape[0]), \n",
    "                                               batch_iterations)\n",
    "\n",
    "                    for i in batch_indices:\n",
    "\n",
    "                        _, train_loss = sess.run([net.train_op, net.regularized_loss], #net.summary_op\n",
    "                                                              feed_dict={net.x_input: x_train[i,np.newaxis,:], \n",
    "                                                                         net.y_input_: y_train[i,np.newaxis,:], \n",
    "                                                                         net.keep_prob:1, net.lr:lr})\n",
    "                        #summary_writer_train.add_summary(summary_str, epoch)\n",
    "                        if(train_loss > 1e9):\n",
    "                            print(\"Step {} | Training Loss: {:.6f}\".format(epoch, train_loss))\n",
    "\n",
    "\n",
    "                    valid_accuracy,valid_loss = sess.run([net.tf_accuracy, net.regularized_loss], #net.summary_op \n",
    "                                                          feed_dict={net.x_input: x_valid[:,np.newaxis,:], \n",
    "                                                                     net.y_input_: y_valid[:,np.newaxis,:], \n",
    "                                                                     net.keep_prob:1, net.lr:lr})\n",
    "                    #summary_writer_valid.add_summary(summary_str, epoch)\n",
    "\n",
    "\n",
    "\n",
    "                    accuracy, pred_value, actual_value, y_pred = sess.run([net.tf_accuracy, \n",
    "                                                                   net.pred, \n",
    "                                                                   net.actual, net.y], \n",
    "                                                                  feed_dict={net.x_input: preprocess.x_test[:,np.newaxis,:], \n",
    "                                                                             net.y_input_: preprocess.y_test[:,np.newaxis,:], \n",
    "                                                                             net.keep_prob:1, net.lr:lr})\n",
    "                    \n",
    "                    accuracy_, pred_value_, actual_value_, y_pred_ = sess.run([net.tf_accuracy, \n",
    "                                                                   net.pred, \n",
    "                                                                   net.actual, net.y], \n",
    "                                                                  feed_dict={net.x_input: preprocess.x_test_[:,np.newaxis,:], \n",
    "                                                                             net.y_input_: preprocess.y_test_[:,np.newaxis,:], \n",
    "                                                                             net.keep_prob:1, net.lr:lr})\n",
    "\n",
    "                    print(\"Step {} | Training Loss: {:.6f} | Train Accuracy: {:.6f} | Test Accuracy: {:.6f}, {:.6f}\".format(epoch, train_loss, valid_accuracy, accuracy, accuracy_))\n",
    "\n",
    "                    if accuracy > Train.best_acc_global:\n",
    "                                Train.best_acc_global = accuracy\n",
    "                                Train.pred_value = pred_value\n",
    "                                Train.actual_value = actual_value\n",
    "                                Train.pred_value_ = pred_value_\n",
    "                                Train.actual_value_ = actual_value_\n",
    "                                Train.best_parameters = \"Hidden Layers:{}, Features Count:{}\".format(h, f)\n",
    "\n",
    "                    if accuracy > Train.best_acc:\n",
    "\n",
    "                        #net.saver.save(sess, \"dataset/tf_vae_only_nsl_kdd_hidden layers_{}_features count_{}\".format(epochs,h,f))\n",
    "                        #Train.results.append(Train.result(epochs, f, h,valid_accuracy, accuracy))\n",
    "                        #curr_pred = pd.DataFrame({\"Attack_prob\":y_pred[:,-2], \"Normal_prob\":y_pred[:, -1]})\n",
    "                        #Train.predictions.update({\"{}_{}_{}\".format(epochs,f,h):curr_pred})\n",
    "\n",
    "                        Train.best_acc = accuracy\n",
    "                        if not (np.isnan(train_loss)):\n",
    "                            net.saver.save(sess, \n",
    "                                       \"dataset/tf_lstm_nsl_kdd/hidden layers_{}_features count_{}/model\"\n",
    "                                       .format(h,f), \n",
    "                                       global_step = epoch, \n",
    "                                       write_meta_graph=False)\n",
    "\n",
    "                        curr_pred = pd.DataFrame({\"Attack_prob\":y_pred[:,-2], \"Normal_prob\":y_pred[:, -1], \"Prediction\":pred_value})\n",
    "                        Train.predictions.update({\"{}_{}_{}\".format(epochs*len(lrs),f,h):\n",
    "                                                  (curr_pred, \n",
    "                                                   Train.result(epochs*len(lrs), f, h,valid_accuracy, accuracy, accuracy_, time.perf_counter() - start_time))})\n",
    "\n",
    "\n",
    "\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-14T18:59:21.362141Z",
     "start_time": "2017-06-14T18:49:00.344078Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Current Layer Attributes - epochs:5 hidden layers:1 features count:122\n",
      "Initial Accuracy, before training: 0.6293026804924011\n",
      "Step 1 | Training Loss: 0.000522 | Train Accuracy: 0.999762 | Test Accuracy: 0.977378, 0.956962\n",
      "Step 2 | Training Loss: 0.000000 | Train Accuracy: 0.999921 | Test Accuracy: 0.980749, 0.963376\n",
      "Step 3 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.980749, 0.963376\n",
      "Step 4 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.980749, 0.963376\n",
      "Step 5 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.980749, 0.963376\n",
      "Step 1 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.980704, 0.963291\n",
      "Step 2 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.980793, 0.963460\n",
      "Step 3 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.980837, 0.963544\n",
      "Step 4 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.980837, 0.963544\n",
      "Step 5 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.980837, 0.963544\n",
      "Step 1 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.980837, 0.963544\n",
      "Step 2 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.980837, 0.963544\n",
      "Step 3 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.980837, 0.963544\n",
      "Step 4 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.980837, 0.963544\n",
      "Step 5 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.980837, 0.963544\n",
      "Current Layer Attributes - epochs:5 hidden layers:2 features count:122\n",
      "Initial Accuracy, before training: 0.3498935401439667\n",
      "Step 1 | Training Loss: 0.005306 | Train Accuracy: 0.995158 | Test Accuracy: 0.972587, 0.949030\n",
      "Step 2 | Training Loss: 0.003534 | Train Accuracy: 0.993174 | Test Accuracy: 0.973740, 0.951224\n",
      "Step 3 | Training Loss: 0.005301 | Train Accuracy: 0.992856 | Test Accuracy: 0.973873, 0.951477\n",
      "Step 4 | Training Loss: 0.005251 | Train Accuracy: 0.998254 | Test Accuracy: 0.996230, 0.992827\n",
      "Step 5 | Training Loss: 0.001767 | Train Accuracy: 0.998809 | Test Accuracy: 0.985185, 0.971814\n",
      "Step 1 | Training Loss: 0.000000 | Train Accuracy: 0.999127 | Test Accuracy: 0.985140, 0.971730\n",
      "Step 2 | Training Loss: 0.001767 | Train Accuracy: 0.998968 | Test Accuracy: 0.985140, 0.971730\n",
      "Step 3 | Training Loss: 0.001767 | Train Accuracy: 0.999047 | Test Accuracy: 0.985140, 0.971730\n",
      "Step 4 | Training Loss: 0.001767 | Train Accuracy: 0.999444 | Test Accuracy: 0.985140, 0.971730\n",
      "Step 5 | Training Loss: 0.000000 | Train Accuracy: 0.999444 | Test Accuracy: 0.985140, 0.971730\n",
      "Step 1 | Training Loss: 0.000000 | Train Accuracy: 0.999444 | Test Accuracy: 0.985140, 0.971730\n",
      "Step 2 | Training Loss: 0.008834 | Train Accuracy: 0.999444 | Test Accuracy: 0.985140, 0.971730\n",
      "Step 3 | Training Loss: 0.000000 | Train Accuracy: 0.999286 | Test Accuracy: 0.985140, 0.971730\n",
      "Step 4 | Training Loss: 0.001767 | Train Accuracy: 0.999206 | Test Accuracy: 0.985140, 0.971730\n",
      "Step 5 | Training Loss: 0.003534 | Train Accuracy: 0.999286 | Test Accuracy: 0.985140, 0.971730\n",
      "Current Layer Attributes - epochs:5 hidden layers:4 features count:122\n",
      "Initial Accuracy, before training: 0.40227997303009033\n",
      "Step 1 | Training Loss: 0.000006 | Train Accuracy: 0.999841 | Test Accuracy: 0.997871, 0.995949\n",
      "Step 2 | Training Loss: 0.000000 | Train Accuracy: 0.999365 | Test Accuracy: 0.997693, 0.995612\n",
      "Step 3 | Training Loss: 0.000001 | Train Accuracy: 0.999127 | Test Accuracy: 0.996363, 0.993080\n",
      "Step 4 | Training Loss: 0.001767 | Train Accuracy: 0.998809 | Test Accuracy: 0.996584, 0.993502\n",
      "Step 5 | Training Loss: 0.000357 | Train Accuracy: 0.998571 | Test Accuracy: 0.962873, 0.929367\n",
      "Step 1 | Training Loss: 0.000002 | Train Accuracy: 1.000000 | Test Accuracy: 0.998714, 0.997553\n",
      "Step 2 | Training Loss: 0.000000 | Train Accuracy: 0.999841 | Test Accuracy: 0.998714, 0.997553\n",
      "Step 3 | Training Loss: 0.000000 | Train Accuracy: 0.999841 | Test Accuracy: 0.998935, 0.997975\n",
      "Step 4 | Training Loss: 0.000207 | Train Accuracy: 1.000000 | Test Accuracy: 0.998935, 0.997975\n",
      "Step 5 | Training Loss: 0.000001 | Train Accuracy: 1.000000 | Test Accuracy: 0.999379, 0.998819\n",
      "Step 1 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999379, 0.998819\n",
      "Step 2 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999379, 0.998819\n",
      "Step 3 | Training Loss: 0.000001 | Train Accuracy: 0.999841 | Test Accuracy: 0.999379, 0.998819\n",
      "Step 4 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999423, 0.998903\n",
      "Step 5 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.999379, 0.998819\n",
      "Current Layer Attributes - epochs:5 hidden layers:6 features count:122\n",
      "Initial Accuracy, before training: 0.606813371181488\n",
      "Step 1 | Training Loss: 0.038772 | Train Accuracy: 0.957612 | Test Accuracy: 0.797640, 0.619409\n",
      "Step 2 | Training Loss: 0.037100 | Train Accuracy: 0.958565 | Test Accuracy: 0.785575, 0.598312\n",
      "Step 3 | Training Loss: 0.043732 | Train Accuracy: 0.962613 | Test Accuracy: 0.767477, 0.560928\n",
      "Step 4 | Training Loss: 0.066175 | Train Accuracy: 0.917685 | Test Accuracy: 0.725515, 0.482194\n",
      "Step 5 | Training Loss: 0.097129 | Train Accuracy: 0.900064 | Test Accuracy: 0.615995, 0.439325\n",
      "Step 1 | Training Loss: 0.094873 | Train Accuracy: 0.916733 | Test Accuracy: 0.664611, 0.369030\n",
      "Step 2 | Training Loss: 0.086229 | Train Accuracy: 0.916495 | Test Accuracy: 0.687944, 0.410042\n",
      "Step 3 | Training Loss: 0.096966 | Train Accuracy: 0.918003 | Test Accuracy: 0.683552, 0.401519\n",
      "Step 4 | Training Loss: 0.074565 | Train Accuracy: 0.914748 | Test Accuracy: 0.684306, 0.402954\n",
      "Step 5 | Training Loss: 0.089134 | Train Accuracy: 0.920464 | Test Accuracy: 0.693666, 0.420422\n",
      "Step 1 | Training Loss: 0.071428 | Train Accuracy: 0.932053 | Test Accuracy: 0.724849, 0.479831\n",
      "Step 2 | Training Loss: 0.057231 | Train Accuracy: 0.931894 | Test Accuracy: 0.738511, 0.506329\n",
      "Step 3 | Training Loss: 0.070697 | Train Accuracy: 0.930862 | Test Accuracy: 0.739221, 0.507257\n",
      "Step 4 | Training Loss: 0.058518 | Train Accuracy: 0.933720 | Test Accuracy: 0.715534, 0.462025\n",
      "Step 5 | Training Loss: 0.080270 | Train Accuracy: 0.922924 | Test Accuracy: 0.699787, 0.432489\n"
     ]
    }
   ],
   "source": [
    "import itertools\n",
    "class Hyperparameters:\n",
    "#    features_arr = [2, 4, 8, 16, 32, 64, 128, 256]\n",
    "#    hidden_layers_arr = [2, 4, 6, 10]\n",
    "    features_arr = [1] #[4, 8, 16, 32]\n",
    "    hidden_layers_arr = [1, 3, 5]\n",
    "\n",
    "    epochs = [5]\n",
    "    lrs = [1e-2, 1e-2/2, 1e-2/4]\n",
    "    \n",
    "    for e, h, f in itertools.product(epochs, hidden_layers_arr, features_arr):\n",
    "        print(\"Current Layer Attributes - epochs:{} hidden layers:{} features count:{}\".format(e,h,f))\n",
    "        n = network(2,h,f)\n",
    "        n.build_layers()\n",
    "        Train.train(e, n, h,f, lrs)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-14T18:59:21.368436Z",
     "start_time": "2017-06-14T18:59:21.363667Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dict1 = {}\n",
    "dict2 = []\n",
    "for k, (v1, v2) in Train.predictions.items():\n",
    "    dict1.update({k: v1})\n",
    "    dict2.append(v2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-14T18:59:21.445750Z",
     "start_time": "2017-06-14T18:59:21.369894Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Train.predictions = dict1\n",
    "Train.results = dict2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-14T18:59:21.473973Z",
     "start_time": "2017-06-14T18:59:21.447367Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_results = pd.DataFrame(Train.results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-14T18:59:21.496820Z",
     "start_time": "2017-06-14T18:59:21.475572Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>epoch</th>\n",
       "      <th>no_of_features</th>\n",
       "      <th>hidden_layers</th>\n",
       "      <th>train_score</th>\n",
       "      <th>test_score</th>\n",
       "      <th>test_score_20</th>\n",
       "      <th>time_taken</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>15</td>\n",
       "      <td>122</td>\n",
       "      <td>4</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.999423</td>\n",
       "      <td>0.998903</td>\n",
       "      <td>174.737208</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>15</td>\n",
       "      <td>122</td>\n",
       "      <td>2</td>\n",
       "      <td>0.998254</td>\n",
       "      <td>0.996230</td>\n",
       "      <td>0.992827</td>\n",
       "      <td>26.940493</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>15</td>\n",
       "      <td>122</td>\n",
       "      <td>1</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.980837</td>\n",
       "      <td>0.963544</td>\n",
       "      <td>28.216267</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>15</td>\n",
       "      <td>122</td>\n",
       "      <td>6</td>\n",
       "      <td>0.957612</td>\n",
       "      <td>0.797640</td>\n",
       "      <td>0.619409</td>\n",
       "      <td>19.484608</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   epoch  no_of_features  hidden_layers  train_score  test_score  \\\n",
       "2     15             122              4     1.000000    0.999423   \n",
       "1     15             122              2     0.998254    0.996230   \n",
       "0     15             122              1     1.000000    0.980837   \n",
       "3     15             122              6     0.957612    0.797640   \n",
       "\n",
       "   test_score_20  time_taken  \n",
       "2       0.998903  174.737208  \n",
       "1       0.992827   26.940493  \n",
       "0       0.963544   28.216267  \n",
       "3       0.619409   19.484608  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_results.sort_values(by = 'test_score', ascending = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-14T18:59:21.515998Z",
     "start_time": "2017-06-14T18:59:21.498385Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pd.Panel(Train.predictions).to_pickle(\"dataset/tf_lstm_nsl_kdd_predictions.pkl\")\n",
    "df_results.to_pickle(\"dataset/tf_lstm_nsl_kdd_scores.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-14T18:59:21.580508Z",
     "start_time": "2017-06-14T18:59:21.518104Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import itertools\n",
    "\n",
    "def plot_confusion_matrix(cm, classes,\n",
    "                          normalize=False,\n",
    "                          title='Confusion matrix',\n",
    "                          cmap=plt.cm.Blues):\n",
    "    \"\"\"\n",
    "    This function prints and plots the confusion matrix.\n",
    "    Normalization can be applied by setting `normalize=True`.\n",
    "    \"\"\"\n",
    "    np.set_printoptions(precision=4)\n",
    "\n",
    "    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n",
    "    plt.title(title)\n",
    "    plt.colorbar()\n",
    "    tick_marks = np.arange(len(classes))\n",
    "    plt.xticks(tick_marks, classes, rotation=45)\n",
    "    plt.yticks(tick_marks, classes)\n",
    "\n",
    "    if normalize:\n",
    "        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n",
    "        print(\"Normalized confusion matrix\")\n",
    "    else:\n",
    "        print('Confusion matrix, without normalization')\n",
    "\n",
    "    print(cm)\n",
    "\n",
    "    thresh = cm.max() / 2.\n",
    "    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n",
    "        plt.text(j, i, cm[i, j].round(4),\n",
    "                 horizontalalignment=\"center\",\n",
    "                 color=\"white\" if cm[i, j] > thresh else \"black\")\n",
    "\n",
    "    plt.tight_layout()\n",
    "    plt.ylabel('True label')\n",
    "    plt.xlabel('Predicted label')\n",
    "\n",
    "def plot(actual_value, pred_value):\n",
    "    from sklearn.metrics import confusion_matrix\n",
    "    cm_2labels = confusion_matrix(y_pred = pred_value, y_true = actual_value)\n",
    "    plt.figure(figsize=[6,6])\n",
    "    plot_confusion_matrix(cm_2labels, preprocess.output_columns_2labels, normalize = False,\n",
    "                         title = Train.best_parameters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-14T18:59:23.357226Z",
     "start_time": "2017-06-14T18:59:21.582263Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion matrix, without normalization\n",
      "[[ 9709     2]\n",
      " [   11 12822]]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAbcAAAGeCAYAAAAXNE8VAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xm8XePZ//HPN4nMM5VGEkKNkZpCKOqnDyWm0pqiZilV\nPKZW8aDoI62ik9ZQfWhMRVBtFE0jrVIkRMQQYwwRERkkMkhEhuv3x7pP7JyeKSf7nH322t+3136d\nve817HvtHOfa17XutW5FBGZmZnnSqtQdMDMzKzYHNzMzyx0HNzMzyx0HNzMzyx0HNzMzyx0HNzMz\nyx0HNzMzyx0HNzMzyx0HNzMzy502pe6AmZkVV+uuG0UsX1K0/cWS2aMjYkjRdtgMHNzMzHImli+h\n3RZHFG1/n066br2i7ayZOLiZmeWOQJV91qmyj97MzHLJmZuZWd4IkErdi5JycDMzyyOXJc3MzPLF\nmZuZWR65LGlmZvni0ZKVffRmZpZLztzMzPLIZUkzM8sV4bJkqTtgZmZWbM7czMxyRy5LlroDZmbW\nBFyWNDMzyxdnbmZmeeSypJmZ5Ysv4q7sozczs1xy5mZmljee8sbBzcwsl1yWNDMzazxJt0iaJenl\ngrarJb0m6UVJD0jqXrDsQklTJL0uad+C9kGSXkrLrpWy9FNSO0n3pPbxkvrX1ycHNzOz3EkDSor1\nqN8IYEi1tjHAwIjYBngDuBBA0gBgKLB12uZ6Sa3TNjcAJwObpUfVPocB8yJiU+CXwM/q65CDm5lZ\nHrVS8R71iIjHgbnV2v4eEcvTy3FA3/T8YODuiFgaEe8AU4DBknoDXSNiXEQEcBtwSME2t6bn9wF7\nVWV1tR5+vb02MzNbOycBj6TnfYBpBcveT2190vPq7attkwLmfGDdut7QA0rMzPKm+LMCrCdpQsHr\nmyLipgZ1RboIWA7cWcwO1cfBzcwsj4p7KcCciNhxzbugE4ADgb1SqRFgOtCvYLW+qW06n5cuC9sL\nt3lfUhugG/BRXe/tsqSZmRWdpCHAD4FvRMTigkWjgKFpBOTGZANHnomIGcACSbuk82nHAX8p2Ob4\n9Pww4B8FwbJGztzMzHKneW+/JekuYE+y8uX7wKVkoyPbAWPS2I9xEXFqREyWNBJ4haxceXpErEi7\nOo1s5GUHsnN0VefpbgZulzSFbODK0Hr7VE/wMzOzMtOqa99oN/iMou3v07EXPteYsmQpOXMzM8sj\n337LzMxyx7ffMjMzyxdnbmZmeSO5LFnqDpiZWRNwWdLMzCxfnLmZmeWRy5JmZpYvzXsRd0tU2Uef\nM5ImS9qzlmV7pjsH1LbtCElXNFnnzMyakYNbmZD0rqS9q7WdIOnfVa8jYuuIeKzZO1eH6n0sJ2l2\n4ZC0aQPX75/WX1TweKEI/bhM0h1ru59ikbS5pHslzZE0P820fG7BhJNN9b71fgGTdIakCZKWShpR\nbdkuksZImitpdjqG3gXLz5P0sqSFkt6RdF4THUrzqBoxWYxHGXJws1xTZo1/zyXtDnypkW/bPSI6\np8e2jdxH0aS7qBdrX18CxpPNrfXliOgGHA4MAroU633WwgfAFcAtNSzrAdwE9Ac2AhYCfyhYXnWz\n3h5kM0CfIaneexi2SFVT3jTfTNwtTnn22mpUmN1J6pC+6c6T9AqwU7V1t5c0MX1LvQdoX235gZIm\nSfpY0lOStqn2Pj9I39jnS7pH0mrbN7C/J0p6NfXhbUnfLVj2sqSDCl6vkzKF7dPrXVK/Ppb0QmE5\nVtJjkoZLehJYDGySMsi3C76VH11Hv9oAvwH+e02PqZ7jPSkd7zxJoyVtVLDs15KmSVog6TlJX03t\nQ4D/AY4szASrZ/KF2V1BBjlM0nvAP1J7XZ9ZQz+fy4GnIuLcdBd3IuL1iDg6Ij5O+/qGshL5x+nf\nYquC91ktEy7MxpRK55K+L2mWpBmSTkzLTgGOBn6YPocHa+pcRPwpIv5MDdOhRMQjEXFvRCxId6n/\nLbBbwfKrImJiRCyPiNfJ7ki/W/X9WHlwcMuvS8kyjy8B+/L5dBFIagv8Gbgd6AncCxxasHx7sm++\n3yWb7fZ3wChJ7Qr2fwTZt9uNgW2AExrRx1lkcz11BU4Efilph7TsNuCYgnX3B2ZExPOS+gAPkX1D\n7wn8ALhf0hcK1j8WOIUsm5gNXAvsFxFdgF2BSelYN0x/hDcs2PYc4PGIeLERx1QjSQeTBalvAV8A\nngDuKljlWWC7dDx/BO6V1D4i/gb8BLinEZng/wO2Avat6zOT1IlaPp8a7A3cV8dxbp6O6+x0nA8D\nD6bfuYb4ItlcXX2AYcB1knqkiTHvBK5Kn8NB6f2ul3R9A/dd3R7A5FqOQ8BXa1ve8smZW6k7YGvk\nz+kP8ceSPgbq+p/6CGB4RMyNiGlkf7yq7AKsA/wqIpZFxH1kf1yrnAL8LiLGR8SKiLgVWJq2q3Jt\nRHwQEXOBB8n+MK+RiHgoIt6KzL+Av5P9QQG4A9hfUtf0+liyYAxZ0Hs4Ih6OiJURMQaYQBYAq4yI\niMlpSvrlwEpgoKQOETEjIianPrwXEd0j4j0ASf3IgvqP1vR4Cswp+Hf6QWo7FfhpRLya+vQTYLuq\n7C0i7oiIj1LW8HOyqUK2WIs+AFwWEZ9ExBLq/8xq/HxqsC4wo473PBJ4KCLGRMQy4Bqy6Ut2bWCf\nlwE/Tr+XDwOLqONziIjTIuK0Bu57lVSJ+BFQ23m1y8j+Pv6hluUtn8+5WRk5JP0h7h4R3cnmPqrN\nBmTnRapMrbZserXJ/gqXbwR8v1og7Ze2q/JhwfPFQOc1ORAASftJGqfsBP/HZH9o1wOIiA+AJ4FD\nJXUH9uPzaeo3Ag6v1r/dgd4Fu1917BHxCdkf3VOBGZIekrRlLd36Fdkf1/lrejwF1iv4d7qmoM+/\nLujvXLIzI33SZ/GDVLKcn5Z3q/os1kLhv3+tn9kafj4fsfrnXN0GFPwuRcTK1I8+DezzRyn4V2nU\n71ZdUln0EeCsiHiihuVnkJ17OyAilhbzva35OLjl1wxWn8p9w2rL+qTSS03Lp5Flfd0LHh0jorCM\ntlZSifN+sm/2vVKwfpjsD36VW8kyjsOBpyOiasr5acDt1frXKSKuLNh2tYkKI2J0RHyd7A/za8Dv\na+naXsDVkj6UVBXAn5b07cYf7ao+f7danztExFPp/NoPybLtHumzmM/nn0VNky5+AnQseP3FGtYp\n3K7Oz2wNPp9HKShh1+ADskAKrCrv9QOq/u0WN6DftVnrySdTpvwo8L8RcXsNy08CLgD2iohaL50p\nCy5LWk6NBC6U1ENSX1YfHPE0WanuTGUDNb4FDC5Y/nvgVEk7K9NJ0gGSGjsaTpLaFz6AtmSlt9nA\nckn7AftU2+7PwA7AWWTn4KrcARwkaV9JrdM+90zHWdOb95J0cDq3tJSs1LWylr5uDmxLVmatKrUe\nBDyQ9nWZpMfW6OgzN5L9e2yd9tNN0uFpWReyf4/ZQBtJPyI7D1llJtBfq4/6nAQMTf9+OwKH1fP+\ntX5ma/j5XArsKulqSV9Mx7KppDtShj0SOEDSXpLWAb6f9vlUQb+/nfowhOy8YEPNBDapawVJbdLv\nV2ug6jjbpGV9yAbX/DYibqxh26PJysVfj4i316BfLZPLkpZTl5OVh94hO5e16ltqRHxGNrDhBLLy\n2JHAnwqWTwBOJhtNNg+YQuMGjFTZFVhSw+NMsj+G84BvA6MKN0rniu4nG7RS2L9pQNUAjdlkWcl5\n1P773Ao4lyyrmEv2B/V7sGpAyaKqASURMSsiPqx6pO3npL5AloU8uaYfQEQ8APwMuFvSAuBlslIr\nwGjgb8AbZP9mn7J6SfHe9PMjSRPT80vIBgvNI/u3/mM971/XZ1br51PDft4CvkI2nH6ypPlk/0YT\ngIVplOExZKNN55B9MTgo/c5B9kXlIOBjstGPf66r39XcDAxIZdU/A0i6UVJhoLqY7HfrgtSPJakN\n4DtkwfEyFVyLWLDtFWTnFJ8tWP4fQdDKg1Y/7WLWsqQsZvOIOKbelZuBpElkJav/GGpu1lK06tE/\n2u15cf0rNtCnfz75uYjYsWg7bAa+t6S1WJJ6kg0HP7bUfakSEWs8KtSsJMq0nFgsLktaiyTpZLLS\n2SMR8Xip+2Nm5cWZm7VIEfF7ah+xZ2b1UIVnbg5uZmY5IxzcXJY0M7PccebWSK07dIs2XdcvdTes\nTAzs263UXbAyMXXqu8yZM2ft0i6x+u0QKpCDWyO16bo+vY/6Zam7YWXiyWsOLHUXrEzstnMxRtzL\nZclSd8DMzKzYnLmZmeVQpWduDm5mZjlU6cHNZUkzM8sdZ25mZjlU6Zmbg5uZWd74UgCXJc3MLH+c\nuZmZ5Yx8nZuDm5lZHlV6cHNZ0szMcseZm5lZDlV65ubgZmaWQ5Ue3FyWNDOz3HHmZmaWN77OzcHN\nzCyPXJY0MzPLGWduZmY544u4HdzMzHKp0oOby5JmZpY7ztzMzPKoshM3Bzczs9yRy5IuS5qZWe44\nczMzy6FKz9wc3MzMcqjSg5vLkmZmljvO3MzMcsYXcTtzMzPLJxXxUd9bSbdImiXp5YK2npLGSHoz\n/exRsOxCSVMkvS5p34L2QZJeSsuuVYrQktpJuie1j5fUv74+ObiZmdnaGgEMqdZ2ATA2IjYDxqbX\nSBoADAW2TttcL6l12uYG4GRgs/So2ucwYF5EbAr8EvhZfR1ycDMzy5t0nVuxHvWJiMeBudWaDwZu\nTc9vBQ4paL87IpZGxDvAFGCwpN5A14gYFxEB3FZtm6p93QfspXo65nNuZmY5VORzbutJmlDw+qaI\nuKmebXpFxIz0/EOgV3reBxhXsN77qW1Zel69vWqbaQARsVzSfGBdYE5tb+7gZmZm9ZkTETs2duOI\nCElRzA7Vx2VJM7Mcas6yZC1mplIj6ees1D4d6FewXt/UNj09r96+2jaS2gDdgI/qenMHNzOzPGrG\n0ZK1GAUcn54fD/yloH1oGgG5MdnAkWdSCXOBpF3S+bTjqm1Tta/DgH+k83K1clnSzMzWiqS7gD3J\nzs29D1wKXAmMlDQMmAocARARkyWNBF4BlgOnR8SKtKvTyEZedgAeSQ+Am4HbJU0hG7gytL4+ObiZ\nmeVQc17EHRFH1bJor1rWHw4Mr6F9AjCwhvZPgcPXpE8ObmZmObOW58pywefczMwsd5y5mZnlUKVn\nbg5uZmY5VOnBzWVJMzPLHWduZmZ5VNmJm4ObmVkeuSxpZmaWM87czMzyRs7cHNzMzHJGQIXHNpcl\nzcwsf5y5mZnljm+/5eBmZpZDFR7bXJY0M7P8ceZmZpZDLkuamVm+yGVJlyXNzCx3nLmZmeWMgFat\nKjt1c+ZmZma548zNzCyHKv2cm4ObmVkOVfpoSZclzcwsd5y5mZnljS8FcHAzM8ubbFaAyo5uLkua\nmVnuOHOzOp24x8YM/Uo/hLh73Hvc8q93+O3xO7DJ+p0A6NphHRYsWcb+Vz8BwGl7f4kjdt6QFRFc\n/qfJPP7abAAO3L43p399M1pL/OOVmVz54GslOyYrrWnTpvGdE49j1qyZSOKkYadwxplnlbpbOeNZ\nARzcrFabf7ELQ7/Sj4N/8W+WrQhu/e5gxk6eyRm3Tly1zkUHb8XCT5cDsGmvzhy0fR/2ufJfrN+t\nHXeetgtfG/5PunZYhwu/MYCDrnmCuZ98xs+/vS27brYuT735UakOzUqoTZs2XHnVz9l+hx1YuHAh\nu+48iL32/jpbDRhQ6q7lSoXHNpclrXab9urMpKkf8+mylaxYGYx/ay5Dtum92joHbLcBo577AIB9\nvtyLB5+fzmcrVvL+3CVMnfMJ223UnQ3X7ci7sz9h7iefAfDvN+aw37a9/+P9rDL07t2b7XfYAYAu\nXbqw5ZZb8cEH00vcK8sbBzer1esfLmSnTXrSveM6tF+nFV8bsD69u7dftXzwJj2Zs3Ap7875BIBe\n3TrwwbxPVy2f8fGn9OrWgXfnLGaT9TvRt2cHWrcS+3z5i/Tu3qHZj8danqnvvsukSc+z0+CdS92V\n3JFUtEc5clnSavXWzEXcOPYtbv/eziz+bAWvTJ/PyohVy78xaANGTfyg3v0sWLKMi+99id8evwMr\nA557Zy4brdepKbtuZWDRokUcdcShXP3zX9G1a9dSdydffClA0wU3SU9FxK5ruM27wHMRcWh6fRhw\nYEScUPwe1tqHy4BFEXFNc71nSzZy/DRGjp8GwHkHbMGMj7PMrHUrse82vTnomidWrTtz/hI26PF5\nZte7e3tmzl8CwNjJsxg7eRYAR31lQ1asbK4jsJZo2bJlHHXEoRx51NEc8s1vlbo7lkNNVpZc08BW\nYJCkRp1ZluRMtMjW7dwWgA26t2fINr0ZNTE7N7L75uvx9sxFfDj/8zLkmJdnctD2fWjbuhV9e3ag\n/3qdmDT149X207XDOhy7+0bcM+69Zj4SaykiglNPHsYWW27FWeecW+ru5FLVdW4uSzYBSYsiorOk\n3sA9QNf0ft+LiCfq2PTnwEXA0dX21xO4BdgEWAycEhEvpkzrS6n9PUmjgUOATsBmwDVAW+BYYCmw\nf0TMlXQycEpaNgU4NiIWF+Xgc+SGEwfRo1Nblq8ILrnvJRYsyUZGHrTDBqsCXZU3P1zEXyd9wJgL\n/x/LVwY/uv9lVqYq5qXf2pqtNshKT9eOfpN3Zn/SrMdhLcdTTz7JH++8nYEDv8zOg7YD4PIrfsKQ\n/fYvcc/ypUxjUtE0R6bzbWB0RAyX1BroWM/6I4HTJG1arf1y4PmIOETSfwG3AdulZQOA3SNiiaQT\ngIHA9kB7ssB1fkRsL+mXwHHAr4A/RcTvASRdAQwDflNXxySdQhYQad3lC/UfeQ4c8Zuna2z/wR9f\nqLH9ujFTuG7MlP9oP/O254vaLytfu+2+O0uWRf0rmq2F5ghuzwK3SFoH+HNETKpn/RXA1cCFwCMF\n7bsDhwJExD8krSup6iz0qIhYUrDuPyNiIbBQ0nzgwdT+ErBNej4wBbXuQGdgdH0HEhE3ATcBtOu1\nmf/vNLMWq1zLicXS5JcCRMTjwB7AdGCEpOMasNntaZt+DXyb6jWupQXPVxa8XsnnAX0EcEZEfJks\nK2yPmVlOSMV7lKMmD26SNgJmphLg/wE71LdNRCwDfgmcU9D8BOk8nKQ9gTkRsWAtutYFmJEyyqPr\nW9nMzMpHc5Ql9wTOk7QMWER2zqshbgYuLnh9GVl580WyASXHr2W/LgHGA7PTzy5ruT8zs5ZBLks2\nWXCLiM7p563ArQ3cpn/B86XABgWv55KNgqy+zWXVXo8gKznWtM9VyyLiBuCG+vZnZlZusksBSt2L\n0vLtt8zMLHdKctGzpPFAu2rNx0bES6Xoj5lZvpTvxdfFUpLgFhG+S6qZWROq8NjmsqSZmeWP78Vo\nZpZDLkuamVm+lPHF18XisqSZmeWOMzczs5ypmvKmkjm4mZnlUKUHN5clzcwsd5y5mZnlUIUnbg5u\nZmZ55LKkmZnZWpB0jqTJkl6WdJek9pJ6Shoj6c30s0fB+hdKmiLpdUn7FrQPkvRSWnat1iJCO7iZ\nmeVNEScqrS+8SOoDnAnsGBEDgdbAUOACYGxEbAaMTa+RNCAt3xoYAlwvqXXa3Q3AycBm6TGksR+B\ng5uZWc4o3Ti5WI8GaAN0kNQG6Ah8ABzM59Od3crnU5YdDNwdEUsj4h1gCjBYUm+ga0SMi4gAbqOG\nac4aysHNzMwaLSKmA9cA7wEzgPkR8XegV0TMSKt9CPRKz/sA0wp28X5q65OeV29vFAc3M7McKnJZ\ncj1JEwoep3z+PupBlo1tTDbBdCdJxxT2JWVi0XxH79GSZma51Kq4oyXnRMSOtSzbG3gnImYDSPoT\nsCswU1LviJiRSo6z0vrTgX4F2/dNbdPT8+rtjeLMzczM1sZ7wC6SOqbRjXsBrwKjgOPTOscDf0nP\nRwFDJbWTtDHZwJFnUglzgaRd0n6OK9hmjTlzMzPLoea6zC0ixku6D5gILAeeB24COgMjJQ0DpgJH\npPUnSxoJvJLWPz0iVqTdnQaMADoAj6RHozi4mZnlTHaurPku4o6IS4FLqzUvJcvialp/ODC8hvYJ\nwMBi9MllSTMzyx1nbmZmOdSqsu++5eBmZpZHvrekmZlZzjhzMzPLoQpP3BzczMzyRmT3l6xkLkua\nmVnuOHMzM8shj5Y0M7N8afhUNbnlsqSZmeWOMzczsxyq8MTNwc3MLG9E0ae8KTsuS5qZWe44czMz\ny6EKT9wc3MzM8sijJc3MzHLGmZuZWc5kk5WWuhel5eBmZpZDHi1pZmaWM7VmbpK61rVhRCwofnfM\nzKwYKjtvq7ssORkIVv+Mql4HsGET9svMzNZCpY+WrDW4RUS/5uyImZlZsTTonJukoZL+Jz3vK2lQ\n03bLzMwaK7v9VvEe5aje4Cbpt8DXgGNT02LgxqbslJmZrYU05U2xHuWoIZcC7BoRO0h6HiAi5kpq\n28T9MjMza7SGBLdlklqRDSJB0rrAyibtlZmZrZUyTbiKpiHB7TrgfuALki4HjgAub9JemZnZWinX\ncmKx1BvcIuI2Sc8Be6emwyPi5abtlpmZWeM19PZbrYFlZKVJ39XEzKwFqxotWckaMlryIuAuYAOg\nL/BHSRc2dcfMzKzxPFqyfscB20fEYgBJw4HngZ82ZcfMzMwaqyHBbUa19dqkNjMza6HKM98qnrpu\nnPxLsnNsc4HJkkan1/sAzzZP98zMbE1JnvKmrsytakTkZOChgvZxTdcdMzOztVfXjZNvbs6OmJlZ\n8VR44lb/OTdJXwKGAwOA9lXtEbF5E/bLzMys0RpyzdoI4A9k5yf3A0YC9zRhn8zMbC1V+qUADQlu\nHSNiNEBEvBURF5MFOTMza6Gk4j3KUUMuBViabpz8lqRTgelAl6btlpmZWeM1JLidA3QCziQ799YN\nOKkpO2VmZo0n5EsB6lshIsanpwv5fMJSMzNrqcq4nFgsdV3E/QBpDreaRMS3mqRHZmZma6muzO23\nzdaLMjSwbzeevObAUnfDykSPnc4odResTCx9/b2i7KdcRzkWS10XcY9tzo6YmVnxVPrcZJV+/GZm\nlkMNnazUzMzKhHBZssHBTVK7iFjalJ0xM7Pi8Ezc9ZA0WNJLwJvp9baSftPkPTMzM2ukhpxzuxY4\nEPgIICJeAL7WlJ0yM7O100rFe5SjhpQlW0XE1Gr12xVN1B8zM1tL2T0hyzQqFUlDgts0SYOBkNQa\n+G/gjabtlpmZWeM1pCz5PeBcYENgJrBLajMzsxaqOcuSkrpLuk/Sa5JelfQVST0ljZH0ZvrZo2D9\nCyVNkfS6pH0L2gdJeiktu1ZrkX7WG9wiYlZEDI2I9dJjaETMaewbmplZ02vmKW9+DfwtIrYEtgVe\nBS4AxkbEZsDY9BpJA4ChwNbAEOD6VBUEuAE4GdgsPYY09vgbMhP376nhHpMRcUpj39TMzPJBUjdg\nD+AEgIj4DPhM0sHAnmm1W4HHgPOBg4G706Vl70iaAgyW9C7QNSLGpf3eBhwCPNKYfjXknNujBc/b\nA98EpjXmzczMrOkJmnPKm42B2cAfJG0LPAecBfSKiBlpnQ+BXul5H2Bcwfbvp7Zl6Xn19kZpyJQ3\n9xS+lnQ78O/GvqGZmTW9It9bcT1JEwpe3xQRN6XnbYAdgP+OiPGSfk0qQVaJiJBU6ywzTaExt9/a\nmM8jsJmZ5d+ciNixlmXvA+8XzP15H1lwmympd0TMkNQbmJWWTwf6FWzfN7VNT8+rtzdKQ+5QMk/S\n3PT4GBgDXNjYNzQzs6bXXANKIuJDskvGtkhNewGvAKOA41Pb8cBf0vNRwFBJ7SRtTDZw5JlUwlwg\naZc0SvK4gm3WWJ2ZW3qDbfk8eq6MiGZNLc3MbM1Ias5zbpBd/3ynpLbA28CJZMnTSEnDgKnAEQAR\nMVnSSLIAuBw4PSKqbgxyGjAC6EA2kKRRg0mgnuCW6qQPR8TAxr6BmZnlW0RMAmoqW+5Vy/rDgeE1\ntE8AihJvGnLOcZKk7YvxZmZm1jya+Tq3FqfWzE1Sm4hYDmwPPCvpLeATslGmERE7NFMfzcxsDZXr\nDY+Lpa6y5DNkwzu/0Ux9MTMzK4q6gpsAIuKtZuqLmZkVQTNfxN0i1RXcviDp3NoWRsQvmqA/ZmZW\nBBUe2+oMbq2BzqQMzszMrFzUFdxmRMSPm60nZmZWHGU8g3ax1HvOzczMyo8q/E94Xde51XjxnZmZ\nWUtXa+YWEXObsyNmZlYc2WjJUveitBozK4CZmbVwlR7cijzlj5mZWek5czMzyyFV+IVuDm5mZjnj\nc24uS5qZWQ45czMzy5synqqmWBzczMxyqNJvnOyypJmZ5Y4zNzOznPGAEgc3M7NcqvCqpMuSZmaW\nP87czMxyR7Sq8FkBHNzMzHJGuCzpsqSZmeWOMzczs7zxTNwObmZmeeSLuM3MzHLGmZuZWc54QImD\nm5lZLrksaWZmljPO3MzMcqjCEzcHNzOzvBEuy1X68ZuZWQ45czMzyxuBKrwu6eBmZpZDlR3aXJY0\nM7MccuZmZpYz2UzclZ27ObiZmeVQZYc2lyXNzCyHnLmZmeVQhVclHdzMzPJHFX8pgMuSZmaWO87c\nzMxyxrffcnAzM8sllyXNzMxyxpmbNcp3v3MSjzz8V76w/vo8N+llAO6/716G/+9lvPbqqzzx1DMM\n2nHHEvfSmtqNlx7NfnsMZPbchex4+E8A+MnZh7D/HgP5bNkK3nl/DqdcegfzFy2hTZtW3PCjo9lu\ny360ad2KOx96hmtu+Tsd2q/DnVcNY5O+67FiZfDw4y9xybWjADjzmP/ihG9+heXLVzJn3iJOvfwO\n3psxr5SHXDYqO29z5maNdOzxJ/CXv/5ttbattx7I3SP/xO5f3aNEvbLmdvuD4zj49OtWaxs77jUG\nHf4TBh/5U96cOovzTtoHgEP33oF2bduw0xE/Ydejf8Z3Dt2NDXv3BOBXt41lu29dwS5Dr+Qr227C\nPrsNAGDSa9PY7eirGHzkT3lg7PMMP+uQ5j3AcpVunFysRzlycLNG2f2re9CzZ8/V2rbcais232KL\nEvXISuFWii6sAAAVz0lEQVTJiW8xd/7i1drGjnuNFStWAvDMS+/Qp1d3AIKgY/u2tG7dig7t2vLZ\nshUs/ORTlny6jMcnvAnAsuUrmPTaNPqsn23z+IQ3WfLpsmxfL767al9m9XFwM7Mmc9zBX2H0k68A\n8KdHn2fxp5/xzpjhvPHIj/nVbWOZt2D1wNitcwf23+PL/POZ1/9jXycc8vm+rG5VoyWL9ShHPudm\nZk3ih8P2ZcWKldz98LMA7LR1f1asWMkm+1xEjy4defSWc/jH+Nd4d/pHALRu3YpbrzyB6+96bFVb\nlaH778QOAzbk69/5dbMfR7kq13JisTRbUJb0VCO3205SSBpS0NZd0mkFr/tL+vZa9O0xSR79YFYk\nxxy0M/vvMZATLhqxqu2I/Xbk70+9wvLlK5k9bxFPT3qbQQM2XLX8uouP4q33ZvPbPz622r6+tvMW\nnD9sXw47+3d8tmx58xyAlb1mC24RsWsjNz0K+Hf6WaU7cFrB6/5Ao4ObmRXP13fdinNP2JvDzv7d\nqvNlAO9/OJc9d8rOyXZs35bB2/Tn9XdnAnDpaQfSrUsHfnD1/avta9st+vLbi4Zy2Dm/Y/a8Rc13\nEDmgIj7KkSKied5IWhQRnSX1Bu4BupKVRb8XEU/Uso2At4CvA08Am0TEp5LuBg4GXgfGAF8FtgLe\nAW4FHgBuBzqlXZ0REU+lfZ4PHAOsBB6JiAskPQb8AJgI3AK8HxEX19CfU4BTAPptuOGgN96aunYf\nShk77pijeOJfjzFnzhzW79WLS350OT169uTcs/+bObNn0717d7bZdjsefHh0qbvaIvTY6YxSd6FJ\n3PrTE/jqoM1Yr3tnZs1dwP/e+DDnnbgP7dq24aP5nwDwzEvvcubwu+nUoS03XX4MW27SGwlu/8s4\nfnnbWPqs350po6/gtbc/ZGnKzG6851+MeOBpHrrxDLbedAM+nLMAgGkfzuPws39XsuNtDktfH8nK\nxbPWKqZsuvW28fO7i/f/3iHb9H4uIuqsbklqDUwApkfEgZJ6kv2t7w+8CxwREfPSuhcCw4AVwJkR\nMTq1DwJGAB2Ah4GzopFBqhTB7ftA+4gYnj6MjhGxsJZtdgN+HBF7SfojcH9E3C+pP/DXiBiY1tsT\n+EFEHJhedwRWpkC4GXBXROwoaT/gEmDviFgsqWdEzE3B7QLgLODliBhe3/EMGrRjPDl+wlp9JlY5\n8hrcrPjKOLidC+wIdE3B7SpgbkRcKekCoEdEnC9pAHAXMBjYAHgU2DwiVkh6BjgTGE8W3K6NiEca\n0+dSDIR5FjhR0mXAl2sLbMlRwN3p+d2sXpqsyzrA7yW9BNwLDEjtewN/iIjFABExt2Cb39HAwGZm\n1pJloyVVtEe97yf1BQ4A/q+g+WCyShrp5yEF7XdHxNKIeAeYAgxOVb2uETEuZWu3FWyzxpo9uEXE\n48AewHRghKTjalovZXWHAj+S9C7wG2CIpC4NeJtzgJnAtmTfJNo2YJungK9Jat+Adc3MKsl6kiYU\nPE6ptvxXwA/JTvdU6RURM9LzD4Fe6XkfYFrBeu+ntj7pefX2Rmn24CZpI2BmRPyeLMrvUMuqewEv\nRkS/iOgfERsB9wPfBBYChUGu+utuwIyIWAkcC7RO7WPIssaOqS+FVyHfTJYGj5TkSyTMrKxJxXsA\ncyJix4LHTZ+/jw4EZkXEc7X1JWVizXMOLClFWXJP4AVJzwNHArVduHIU2cCQQvcDR0XER8CTkl6W\ndDXwIrBC0guSzgGuB46X9AKwJfAJQET8DRgFTJA0iWwQySoR8QvgeeB2SeV67aKZVTwV9b967AZ8\nI1XY7gb+S9IdwMxUaiT9nJXWnw70K9i+b2qbnp5Xb2/cJ9BcA0ryxgNKbE14QIk1VDEGlGy29Xbx\nq3v+XqwuceCXe9U7oARWH9yXEo+PCgaU9IyIH0raGvgjnw8oGQtsVsuAkt9ExMON6bPLb2ZmOdQC\nblByJdlpnmHAVOAIgIiYLGkk8AqwHDg9IlakbU7j80sBHkmPRmkRwU3SeKBdteZjI+KlUvTHzKyc\nVY2WbG4R8RjwWHr+EdnYiZrWGw78x8j0iJgADCxGX1pEcIuInUvdBzMzy48WEdzMzKyI1CLKkiXl\n4GZmlkOVHtw83N3MzHLHmZuZWQ414Pq0XHNwMzPLGQGtKju2uSxpZmb548zNzCyHXJY0M7Pc8WhJ\nMzOznHHmZmaWQy5LmplZrni0pMuSZmaWQ87czMxyp0GTjOaag5uZWd74xskuS5qZWf44czMzy6EK\nT9wc3MzM8iYbLVnZ4c1lSTMzyx1nbmZmOVTZeZuDm5lZPlV4dHNZ0szMcseZm5lZDvkibjMzy50K\nHyzpsqSZmeWPMzczsxyq8MTNwc3MLJcqPLq5LGlmZrnjzM3MLGeER0s6uJmZ5Y2nvHFZ0szM8seZ\nm5lZDlV44ubgZmaWSxUe3VyWNDOz3HHmZmaWO/JoyVJ3wMzMis+jJc3MzHLGmZuZWc6Iih9P4uBm\nZpZLFR7dXJY0M7PcceZmZpZDHi1pZma549GSZmZmOePMzcwshyo8cXNwMzPLHV8L4LKkmZnljzM3\nM7Mc8mhJMzPLFeHRki5LmplZ7jhzMzPLoQpP3BzczMxyqcKjm8uSZmbWaJL6SfqnpFckTZZ0Vmrv\nKWmMpDfTzx4F21woaYqk1yXtW9A+SNJLadm1UuPPHDq4mZnlkIr4Xz2WA9+PiAHALsDpkgYAFwBj\nI2IzYGx6TVo2FNgaGAJcL6l12tcNwMnAZukxpLHH7+BmZpZDUvEedYmIGRExMT1fCLwK9AEOBm5N\nq90KHJKeHwzcHRFLI+IdYAowWFJvoGtEjIuIAG4r2GaNObiZmVlRSOoPbA+MB3pFxIy06EOgV3re\nB5hWsNn7qa1Pel69vVE8oMTMLIeKPJ5kPUkTCl7fFBE3rfZ+UmfgfuDsiFhQeLosIkJSFLdLdXNw\nMzPLo+JGtzkRsWOtbyWtQxbY7oyIP6XmmZJ6R8SMVHKcldqnA/0KNu+b2qan59XbG8VlSTMza7Q0\novFm4NWI+EXBolHA8en58cBfCtqHSmonaWOygSPPpBLmAkm7pH0eV7DNGnPmZmaWM9mkAM12odtu\nwLHAS5Impbb/Aa4ERkoaBkwFjgCIiMmSRgKvkI20PD0iVqTtTgNGAB2AR9KjURzczMzypgGjHIsl\nIv5N7UXQvWrZZjgwvIb2CcDAYvTLZUkzM8sdZ26NNHHic3M6rKOppe5HC7MeMKfUnbCy4d+Xmm1U\njJ1U+N23HNwaKyK+UOo+tDSSJtQ1osqskH9fmliFRzeXJc3MLHecuZmZ5U6D7gmZaw5uVkw31b+K\n2Sr+fWlCnonbrEiq347HrC7+fbGm5MzNzCxnRMWPJ3FwMzPLpQqPbi5LmplZ7jhzs5KS1BNYLyLe\nKHVfrHxIUprQ0mpR6aMlnblZyUhqD5wJnCRpq1L3x1o+Sf0gmx+s1H1p6ZprJu6WysHNSiYiPgUe\nTS8PlzSglP2xlkdSZ0lt0/OtgKskdSlxt6wMOLhZSaT5mqruKD4K6Aoc5gBnVSR1Au4EDk9Ni9Nj\nUZocc9Xvkf0nFfFRjhzcrNlVnS+RtLGkNhHxFPAHoBtZgHOJ0oiIT4B7gBMlHQn0B5ZEZllax+VJ\nq5EHlFizS4HtAOAS4AlJi4Bfkd2xYhhwjKQ7I+KVUvbTSkdS64hYERF/lDQbOB94DthY0q+B94Gl\nQJtqsz8bNOt8bi2VMzdrdpJ2AX4CHEn2BesQ4CpgNnAr0An4rGQdtJJKmf0KSV+XdFVEjAF+TTbx\n5WfAe+lnZ2B8CbvawlV2YdKZmzUbSa2AIJvH6zhgS2AP4ALgFOAasm/oF6WSlFWglNnvBVwPfDe1\nPShpOXAu8EZEPFjKPlrL58zNmlzBSf/O6XzJXyPiBbKM7TsRMRqYRfZlq5cDW+VSpg0wBLgkIv5R\nNVoyIh4BbgTOl9SnlP1s6YQvBXBwsyZXcI5trKTLJH0rLVofOEXSzsBg4JqIeLlkHbWSS19+lgOf\nArtIah8RnwFI2gl4GPhGREwvZT/LQWUXJR3crBlI6g0cTVZ2nAvsm4LdSUA/4EfATyPixdL10kql\nKrOXtKGkvqn5EWAd4P+lZdsCvwQ2j4i5JemolRWfc7MmJWlHYFtgekTcI+kLwL7AN4F1IuJASR0j\nYrFvqVSZCjL7nwJPSeoZEUekS0KOlXQ+2WUiV6RytjVAuZYTi8XBzZqMpD3JRj+OJhvef1dETJT0\nCNAWOFjSMxHxAfiapUpTcL3jLmSjZQ8ky9RukfRoROwtaQTZl6P5EfGWvwA1XKXfW9LBzZqEpI2B\n/wGOjYjHJU0B7pB0dEQ8L+kvwN+qAptVjnRP0WVpuH8v4CPgCGAzstGR3YDHJD0VEbsCE6u2dWCz\nhvI5NyuagnMnO5F9A+9GNiKSiLgKuBkYJWlQRHzkwFZ50uUguwJnSzqQ7HzrQuAV4ADglohYSJbx\nb5h+l6wxKnxEiYObFU0qMe1BVmJ6iexC7Y6SzkjLfw5cR3bxrVWuF4F9gNuB+yLiQ7I/oTOAL0k6\nmaxE+fWIeLZ03SxvFR7bHNyseCRtAXwPGBERzwGPAWOBLSV9HyAiroyIf/mGt5VFUidJfSNiJbBR\nav4nsF8a7r+SbIaIxWSB7caIeLVE3bUc8Dk3K6YvA72AvSU9HBGzJf2NbEj3npI2ioip4HMnFag/\ncIWkCcBA4PvAPLL7i/4COA14myzg/SQilnvwSOOV88XXxeLMzRqt4BxbX0ndIuI+sj9WC8ju7r9u\nOn/yIPCjqsBmlSciJgNTyAYZjU8X688mu8VWO0ljyTL9Zekibn8BWksq4n/lyMHNGkVSq3SObT+y\nC25vlvQ48CrwV6DqGqV1I2JhOq9iFURSd0kdC5peBn4OHCdpr4j4LF24fxEwAjgnIsaVoKuWQy5L\n2hqR1CEilkTESkmbAv8LfDcinpJ0LfBnsou010k/O5EN9bYKIqkn8AbwqKQnIuK6iLg1LZsG/ELS\n8cDHwLeqpq1xKbKIyjPhKhoHN2swSd2AKyU9EBF/J/vD9BrZHzEi4kxJdwEXRMSlkp6NiBkl7LKV\nzjzg72QjII+WNBj4N3BvRPxe0mfA/cBy4OyqjRzYiqfCY5vLkrZGupKdN/l2mpJkAbAusHfBOg+T\n5mJzYKtcKUhNJBtgtAdZ2XEP4F+SvkY2cGRn4NB0t3+zonLmZvWS1CWdN5sm6TZgKNlNj2eTDRAY\nIWlLYH5q/2HpemstRURcI+lhsi8/LwPbkWX6Q4FNgSM9C0TTqfTRkg5uVidJ/YH7JD0HjATeBP4A\nLCUbzv0z4HBgP2ADskEBj/rcSWWT1DoiVpBlbN8ku6P/zSngrU920+w5pexjvpXvKMdicXCz+rQH\negMHA++S3WHkRqAH8BTZ0P/hEfHrwo0c2CpbCmwA44HLgKcj4prUNtu/H9bUfM7NapWG+79GVlaa\nD7wHHAl8QHbvyMPS66vSsG//PtkqKXufCpwLdK6aPduBrel5Jm5nblaHNNy/VUS8KukY4G6yu0fc\nLOk+sru4HwxMioiPS9pZK4mCaWtapVtorVIQxN4HVv7n1mZNx8HN6lQQ4J6VNBS4K90L8DrgdbKb\nJPv6pApUENj2IsvMRkfEp9XXi4iXJZ0fEdNL0E2rUC4jWb0KAxxZGfISSadXW8eBrYKkASMhaQhw\nAzCvpsCmTKuImCqpo6R1m7+3lanSy5IObrZKwb0i/+P3oiDAPQccBExu7v5Z6UnaNF0askJSD7IB\nRaemCWm/Kun4dMF2lVbpd6c72bVtPUvS8QpU6feWdFnSgIaVmKplcC5FVqZewPqSxkXEPEn/BIal\nOdhaAcvIzsU+I6lNurt/N+Be4LyIeLN0XbdK4szNGlxiqlo9bdOB7HIAqyAR8STZRLRvS+pKdh3b\nM8BvIuJIsmsht5bUNgW2HsADwI8j4vFS9bviFLEk6bKklZ01LTFVXZibSkyPkd16yypMmsboLLLr\nHOdExK/TjbO/SnYj7f+LiM/S6kcBV0TEEyXqbkUq5izcZRrbXJascC4xWaNExF8kLQOekzQI+JTs\nuseLI+KhqpJ1RFxf2p5apXJwq2AR8aSkLmQlpm3ISkwHAM+mb+LfAE5MJabPUnZ3P3Cpv4lbRDws\naSXZHH5bAOdHxKcF5299TraUyjXlKhKXJSucS0y2NiLib8B3gO2rztNWBTQHttLyaEmreC4x2dqI\niIfAo2etZXFwM8AlJlt7/v1oWcp1lGOxuCxpq7jEZJYfHi1pVsAlJjPLAwc3q5EDm1mZK9eUq0gc\n3MzMcqhcRzkWi8+5mZlZ7jhzMzPLmaqZuCuZfGrF8kbSCrKb+7Yhu7Th+IhY3Mh97Qn8ICIOTHds\nGRARV9aybnfg22t6PaCky4BFEXFNQ9qrrTMC+GtE3NfA9+qf1h+4Jn208iLpb8B6RdzlnIgYUsT9\nNTlnbpZHSyJiOwBJdwKnAr+oWpjmrVNErFyTnUbEKGBUHat0B04DfLG7lVS5BaKm4HNulndPAJtK\n6i/pdUm3AS8D/STtI+lpSRMl3SupM4CkIZJekzQR+FbVjiSdIOm36XkvSQ9IeiE9dgWuBL4kaZKk\nq9N650l6VtKLki4v2NdFkt6Q9G+yi+brJOnktJ8XJN0vqWPB4r0lTUj7OzCt31rS1QXv/d21/SDN\nyomDm+WWpDbAfmQlSshmOLg+IrYGPgEuBvaOiB2ACcC5ktoDvyebbXwQ8MVadn8t8K+I2BbYgWxm\n8guAtyJiu4g4T9I+6T0HA9sBgyTtkW5xNjS17Q/s1IDD+VNE7JTe71VgWMGy/uk9DgBuTMcwDJgf\nETul/Z8saeMGvI9ZLrgsaXnUQdKk9PwJ4GZgA2BqRIxL7bsAA4AnsyolbYGngS2Bd6qm85F0B3BK\nDe/xX8BxABGxApifZk0otE96PJ9edyYLdl2AB6rOA0qqq9RZZaCkK8hKn52B0QXLRqYS65uS3k7H\nsA+wjaTD0jrd0nu/0YD3Mit7Dm6WR6vOuVVJAeyTwiZgTEQcVW291bZbSwJ+GhG/q/YeZzdiXyOA\nQyLiBUknAHsWLKs+KizSe/93RBQGwaoBJWa557KkVapxwG6SNgWQ1EnS5sBrQH9JX0rrHVXL9mOB\n76VtW6dJXBeSZWVVRgMnFZzL6yNpfeBx4BBJHdJ8egc1oL9dgBmS1gGOrrbscEmtUp83AV5P7/29\ntD6SNpfUqQHvY5YLztysIkXE7JQB3SWpXWq+OCLekHQK8JCkxWRlzS417OIs4CZJw4AVwPci4mlJ\nT0p6GXgknXfbCng6ZY6LgGMiYqKke4AXgFnAsw3o8iXAeGB2+lnYp/eAZ4CuwKlpNof/IzsXNzGN\nDp0NHNKwT8es/Pk6NzMzyx2XJc3MLHcc3MzMLHcc3MzMLHcc3MzMLHcc3MzMLHcc3MzMLHcc3MzM\nLHcc3MzMLHf+P9f/bkGSogSyAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f4be067f8d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot(actual_value = Train.actual_value, pred_value = Train.pred_value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-14T18:59:23.630470Z",
     "start_time": "2017-06-14T18:59:23.358987Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion matrix, without normalization\n",
      "[[2150    2]\n",
      " [  11 9687]]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAbUAAAGgCAYAAAAtsfn1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xm8XePZ//HPN4lISCKJMSJEzRFDEtODalpT1FitiCmK\nmrWGammV0ofyU23RFqVI0CLmqJk+ah5CgoSaBREkYkiEyHD9/lj3SXeOc07OSfY5e++1vm+v/Tp7\nr/FaO9u+9nXf91pLEYGZmVketKt0AGZmZuXipGZmZrnhpGZmZrnhpGZmZrnhpGZmZrnhpGZmZrnh\npGZmZrnhpGZmZrnhpGZmZrnRodIBmJlZ62rfbbWIOV+UbXvxxZR7ImJI2TZYRk5qZmY5F3O+YMl1\nhpZte1+O+8tyZdtYmTmpmZnlnkDF6G0qxlGamVkhuFIzM8s7AVKlo2gTTmpmZkXg5kczM7Pa4krN\nzKwI3PxoZmb54NGPZmZmNceVmplZEbj50czMckG4+dHMzKzWuFIzM8s9ufnRzMxyxM2PZmZmtcWV\nmplZEbj50czM8sEnX5uZmdUcV2pmZnnnW8+YmVmuuPnRzMystrhSMzPLveIMFHFSMzMrgnbF6FMr\nRuo2M7NCcKVmZpZ3vkq/mZlZ7XGlZmZWBD5PzczM8qE4ox+LcZRmZlYIrtTMzIrAzY9mZpYbbn40\nMzOrLa7UzMzyTnLzo5mZ5YibH83MzGqLk1oOSZogaXAj8wZLereJdUdIOrPVgjOzyqhrgizHo4o5\nqdUYSW9J2q7etB9KeqTudUSsHxEPtnlwTagfYy2RdIWkkLRmM5fvm5afUfJ4rgxxnC7pmsXdTrlI\nWlvSDZKmSvpU0vOSTpDUvpX3u9AfXpKOkTRG0ixJI+rN20LSfZKmSZqSjqFXyfyfSRovabqkNyX9\nrJUOpQ2lk6/L9ahi1R2dWZko0+LPu6StgTUWcbfdI6JLemy0iNsoG0ll60OXtAbwJPAOsEFELAPs\nBQwCupZrP4vhPeBM4IoG5vUALgX6AqsB04ErS+YLGJ6WGwIcI2lYawZr5eOklkOl1ZykzumX7ceS\nXgQ2rbfsAEnPpl+l1wOd6s3fRdI4SZ9IekzShvX2c2L6hf6ppOslLbB+M+M9SNJLKYY3JB1eMm+8\npF1LXi+RKoMB6fUWKa5PJD1X2uwq6UFJZ0l6FJgJfCNVjG+U/Arfr4m4OgB/An7c0mNayPEenI73\nY0n3SFqtZN4Fkt6R9JmkZyR9M00fAvwS2Lu08qtfuZdWcyUV4yGS3gb+laY39Z419/05A3gsIk6I\niMkAEfFyROwXEZ+kbe2mrCn8k/RvsV7JfhaofEurL6Umckk/lfShpMmSDkrzDgP2A36e3ofbGwou\nIm6OiFuBjxqYd1dE3BARn0XETODPwFYl88+NiGcjYk5EvAzcVjq/Zrn50XLi12SVxhrAjsCBdTMk\ndQRuBa4GegI3AN8vmT+A7Jfu4cCywF+B0ZKWLNn+ULJfs6sDGwI/XIQYPwR2AboBBwF/lDQwzbsK\n2L9k2e8CkyNirKTewB1kv8h7AicCN0lavmT5A4DDyKqHKcCFwE4R0RXYEhiXjnXV9OW7asm6xwMP\nRcTzi3BMDZK0O1ly2hNYHngYuLZkkaeBjdPx/AO4QVKniLgb+C1w/SJUft8C1gN2bOo9k7Q0jbw/\nDdgOuLGJ41w7Hddx6TjvBG5Pn7nmWAlYBugNHAL8RVKPiLgU+Dtwbnofdk37u0jSRc3cdn3bABMa\nOQ4B32xsfs2ou/WMmx+tSt2avoA/kfQJ0NT/zEOBsyJiWkS8Q/alVWcLYAng/IiYHRE3kn2p1jkM\n+GtEPBkRcyNiJDArrVfnwoh4LyKmAbeTfSG3SETcERGvR+bfwL1kXyQA1wDfldQtvT6ALAlDluzu\njIg7I2JeRNwHjCFLfHVGRMSEiJgDzAHmAf0ldY6IyRExIcXwdkR0j4i3AST1IUvmp7X0eEpMLfl3\nOjFNOwI4OyJeSjH9Fti4rlqLiGsi4qNUJfweWBJYZzFiADg9Ij6PiC9Y+HvW4PvTgGWByU3sc2/g\njoi4LyJmA+cBnckSZXPMBn6TPpd3AjNo4n2IiKMi4qhmbnu+1PJwGtBYv9npZN+TVzYy36qMk1pt\n2iN9AXePiO5AU/8zr0zW71FnYr15kyIiGpm/GvDTegm0T1qvzvslz2cCXVpyIACSdpL0hLKO+0/I\nvmCXA4iI94BHge9L6g7sRPZLvS6+verFtzXQq2Tz8489Ij4n+7I9Apgs6Q5J6zYS1vlkX6qftvR4\nSixX8u90XknMF5TEO43sd3Tv9F6cmJomP03zl6l7LxZD6b9/o+9ZC9+fj1jwfa5vZUo+SxExL8XR\nu5kxf5SSfp1F+mw1JTV/3gUcGxEPNzD/GLK+tZ0jYlY59932PFDE8mMyWSKqs2q9eb1TE0tD898h\nq/K6lzyWiojS5rLFkpoybyL7Jb9iStJ3kn3R1xlJVmHsBTweEZNK4ru6XnxLR8Q5JeuWJmwi4p6I\n2J7sC/k/wGWNhLYt8DtJ70uqS9yPS9p30Y92fsyH14u5c0Q8lvrPfk5WXfdI78Wn/Pe9iAa29zmw\nVMnrlRpYpnS9Jt+zFrw/91PSVN2A98gSKDC/Ga8PUPdvN7MZcTemofehRVJlfD/wvxFxdQPzDwZO\nBraNiEZPgakp7lOznBgF/EJSD0mrsOCgh8fJmuR+omwAxp7AZiXzLwOOkLS5MktL2lnSoo5uk6RO\npQ+gI1kT2xRgjqSdgB3qrXcrMBA4lqyPrc41wK6SdpTUPm1zcDrOhna+oqTdU9/RLLImrXmNxLo2\nsBFZc2pdk+quwC1pW6dLerBFR5+5hOzfY/20nWUk7ZXmdSX795gCdJB0Glk/Y50PgL5acBTnOGBY\n+vfbBPjBQvbf6HvWwvfn18CWkn4naaV0LGtKuiZV1KOAnSVtK2kJ4Kdpm4+VxL1vimEIWb9fc30A\nfKOpBSR1SJ+v9kDdcXZI83qTDZr5c0Rc0sC6+5E1C28fEW+0IC6rAk5q+XcGWTPQm2R9VfN/lUbE\nV2QDFn5I1gy2N3BzyfwxwKFko8M+Bl5j0QaC1NkS+KKBx0/IvgQ/BvYFRpeulPqCbiIbjFIa3ztA\n3cCLKWRVyM9o/HPdDjiBrIqYRvZFeiTMHygyo26gSER8GBHv1z3S+lNTLJBVHY+29A2IiFuA/wdc\nJ+kzYDxZkyrAPcDdwCtk/2ZfsmDT4Q3p70eSnk3PTyUbBPQx2b/1Pxay/6bes0bfnwa28zrwP2TD\n4idI+pTs32gMMD2NGtyfbPToVLIfBLumzxxkP1B2BT4hG814a1Nx13M50C81n94KIOkSSaUJ6ldk\nn62TUxxfpGkAPyJLiqer5FzCknXPJOszfLpk/teSX80pSPOjFuxOMatOqWpZOyL2X+jCbUDSOLKm\nqa8NGTerNu26rxZLDj6lbNv78rbDn4mITcq2wTLyBY2t6knqSTas+4BKx1InIlo8ytPMWl9115FW\neJIOJWsiuysiHqp0PGY1ScUZ/ehKzapaRFxG4yPwzKy5qnzUYrlUd8o1MzNrAVdqZmYFoIJUak5q\ni6h7z2Vj5d6rLnxBsxKdO7bqXVkshyZOfIupU6cuVkYSTmq2ECv3XpWrRj9Y6TCsxvTvs0ylQ7Aa\ns9XmVTlyvmo5qZmZ5Z1Y8MJzOeakZmaWeypM86NHP5qZWW64UjMzK4CiVGpOamZmBVCUpObmRzMz\nyw1XamZmBVCUSs1Jzcws7wo0pN/Nj2Zmlhuu1MzMck4+T83MzPJEUtkezdjX8ZImSBov6VpJnST1\nlHSfpFfT3x4ly/9C0muSXpa0Y8n0QZJeSPMuVDN27qRmZmZlI6k38BNgk4joD7QHhgEnAw9ExFrA\nA+k1kvql+esDQ4CLJNVd+fti4FBgrfQYsrD9O6mZmRVAW1ZqZF1bnSV1AJYC3gN2B0am+SOBPdLz\n3YHrImJWRLwJvAZsJqkX0C0inoiIAK4qWafJHZuZWc61VZ9aREySdB7wNvAFcG9E3CtpxYiYnBZ7\nH1gxPe8NPFGyiXfTtNnpef3pTXKlZmZmLbWcpDElj8PqZqS+st2B1YGVgaUl7V+6cqq8ojUCc6Vm\nZpZ35T9PbWpENHajt+2ANyNiCoCkm4EtgQ8k9YqIyalp8cO0/CSgT8n6q6Rpk9Lz+tOb5ErNzKwA\n2rBP7W1gC0lLpdGK2wIvAaOBA9MyBwK3peejgWGSlpS0OtmAkKdSU+VnkrZI2xlesk6jXKmZmVnZ\nRMSTkm4EngXmAGOBS4EuwChJhwATgaFp+QmSRgEvpuWPjoi5aXNHASOAzsBd6dEkJzUzs5xr65Ov\nI+LXwK/rTZ5FVrU1tPxZwFkNTB8D9G/Jvp3UzMwKwFcUMTMzqzGu1MzMiqAYhZqTmplZ7snNj2Zm\nZjXHlZqZWQEUpVJzUjMzK4CiJDU3P5qZWW64UjMzy7ki3fnaSc3MrAiKkdPc/GhmZvnhSs3MLO8K\ndJ6ak5qZWQEUJam5+dHMzHLDlZqZWQEUpVJzUjMzK4Ji5DQ3P5qZWX64UjMzKwA3P5qZWS5Ixbmi\niJsfzcwsN1ypmZkVQFEqNSc1M7MCKEpSc/OjmZnlhis1M7MiKEah5qRmZlYEbn40MzOrMa7UzMzy\nzreeMTOzvBBQkJzm5kczM8sPV2pmZrlXnMtkOamZmRVAQXKamx/NzCw/XKmZmRVAUZofXamZmVlu\nuFIzM8s7FadPzUnNzCznBLRrV4ys5uZHMzPLDVdqZmYF4OZHMzPLDY9+NDMzqzGu1MzM8s6jH83M\nLC+yq/QXI6u5+dGa9P5773LEvrswdIfNGbrjFlx75cUA3H/nrQzdcQs2W6MHLz4/dv7y7707ka3X\nW4l9d96afXfemrNPOX7+vJdeGMewIVvyvW8P4Lwzfk5EtPnxWHV555132HG7bzNgw34M3Gh9/nzh\nBZUOyWqcKzVrUocOHTjul2eybv+N+XzGdIbvNpjNt/42a6y9HudefDVnn3Lc19bpvdrq/OOOR742\n/ZxTT+CUsy+g/8abcOzBe/HYv+9nq8Hbt8VhWJXq0KED55z7ewYMHMj06dPZcvNBbLvd9qzXr1+l\nQ8uZ4lyl35WaNWm5FVZi3f4bA7B0l670XXNtprw/mdXXXIe+31ir2duZ+uH7fD5jOhsM2BRJ7Py9\nYfz7vjtaK2yrEb169WLAwIEAdO3alXXXXY/33ptU4ajySSrfo5o5qVmzvffuRF6e8ALrbzyo6eXe\nmci+O2/NYcO+y9inHgPgw/cns8JKK89fZoWVVmbK+5NbNV6rLRPfeotx48ay6WabVzoUq2FufrRm\nmfn5DE46ajgnnPpbunTt1uhyyy2/Erc/Mp7uPXry0gvjOPGI/bj+7sfbMFKrRTNmzGCfod/nd78/\nn27dGv982aJz8+NikvTYIqzzlqSbSl7/QNKIsga28BhOl3RiW+6z2s2ZPZuTjhrOkN324jtDdmty\n2Y5LLkn3Hj0BWG+DjVll1b68/ebrrLBSLz58/735y334/nssv1KvVo3basPs2bPZZ+j32Xuf/djj\ne3tWOpx8KmPTY7XnxlZLahGx5SKuOkjSIvUSS3LlWWYRwf+efAx911ib/X50zEKX//ijqcydOxeA\nd99+i3feeoPeq/ZluRVWYukuXXlh7NNEBHfcch3f2u67rR2+VbmI4IhDD2Gdddfj2ONPqHQ4lgOt\nlgQkzYiILpJ6AdcD3dL+joyIh5tY9ffAKcB+9bbXE7gC+AYwEzgsIp6XdDqwRpr+tqR7gD2ApYG1\ngPOAjsABwCzguxExTdKhwGFp3mvAARExsywHnyPPjXmCO2+5njXX6ce+O28NwNEnnsZXX83ivDNO\n4uNpUzn+kKGs3W8D/jTyZsY+9SiXnH82HTp0oF27dpx85h9YpnsPAE76ze854+dHMevLL9jyW9uz\npUc+Ft5jjz7KP/5+Nf37b8Dmg7IBSWec+VuG7OQfPOVUpPPU2qKy2Re4JyLOktQeWGohy48CjpK0\nZr3pZwBjI2IPSd8BrgI2TvP6AVtHxBeSfgj0BwYAncgS1kkRMUDSH4HhwPnAzRFxGYCkM4FDgD81\nFZikw8gSISut3GfhR54DG2/6Pzz9xicNzvv2jrt+bdp3dtqd7+y0e4PL99twgPvXbAFbbb01X8z2\n+YptoSA5rU1GPz4NHJQqqg0iYvpClp8L/A74Rb3pWwNXA0TEv4BlJdX1KI+OiC9Klv2/iJgeEVOA\nT4Hb0/QXgL7peX9JD0t6gawqXH9hBxIRl0bEJhGxSY+eyy5scTMza2OtntQi4iFgG2ASMELS8Gas\ndnVap7nl0Of1Xs8qeT6v5PU8/ludjgCOiYgNyKrATs3cl5lZzZFUtkc1a/WkJmk14IPU1Pc3YODC\n1omI2cAfgeNLJj9M6meTNBiYGhGfLUZoXYHJkpagXv+dmVneFGX0Y1v0qQ0GfiZpNjCDrE+rOS4H\nflXy+nTgCknPkw0UOXAx4zoVeBKYkv52XcztmZlZhbVaUouILunvSGBkM9fpW/J8FrByyetpZKMa\n669zer3XI8iaFhva5vx5EXExcPHCtmdmVvPk0Y9mZpYT2ZD+SkfRNiqS1CQ9CSxZb/IBEfFCJeIx\nM7N8qEhSiwhfsdTMrM1U/6jFcnHzo5lZARQkp/nWM2Zmlh+u1MzMCsDNj2Zmlg81cNJ0ubj50czM\ncsOVmplZzvnWM2ZmlitFSWpufjQzs9xwpWZmVgAFKdSc1MzMisDNj2ZmZjXGlZqZWd75PDUzM8sL\npQsal+vRrH1K3SXdKOk/kl6S9D+Sekq6T9Kr6W+PkuV/Iek1SS9L2rFk+iBJL6R5F2ohATipmZlZ\na7gAuDsi1gU2Al4CTgYeiIi1gAfSayT1A4YB6wNDgIsktU/buRg4FFgrPYY0tVMnNTOzApDK91j4\nvrQMsA1wOUBEfBURnwC7AyPTYiOBPdLz3YHrImJWRLwJvAZsJqkX0C0inoiIAK4qWadB7lMzMyuA\nduXtVFtO0piS15dGxKUlr1cHpgBXStoIeAY4FlgxIianZd4HVkzPewNPlKz/bpo2Oz2vP71RTmpm\nZtZSUyNikybmdwAGAj+OiCclXUBqaqwTESEpyh2Ymx/NzAqgLZsfySqqdyPiyfT6RrIk90FqUiT9\n/TDNnwT0KVl/lTRtUnpef3qjnNTMzHIuS0ZtN/oxIt4H3pG0Tpq0LfAiMBo4ME07ELgtPR8NDJO0\npKTVyQaEPJWaKj+TtEUa9Ti8ZJ0GufnRzMxaw4+Bv0vqCLwBHERWSI2SdAgwERgKEBETJI0iS3xz\ngKMjYm7azlHACKAzcFd6NMpJzcysANq18cnXETEOaKjfbdtGlj8LOKuB6WOA/s3dr5OamVkB+NqP\nZmZmNcaVmplZARSkUHNSMzPLO5Fd/7EI3PxoZma54UrNzKwA2nr0Y6U4qZmZ5V0LbhlT69z8aGZm\nueFKzcysAApSqDmpmZnlnSj7rWeqlpsfzcwsN1ypmZkVQEEKNSc1M7Mi8OhHMzOzGuNKzcws51pw\nx+qa56RmZlYAHv1oZmZWYxqt1CR1a2rFiPis/OGYmVlrKEad1nTz4wQgWPC9qHsdwKqtGJeZmZVR\nUUY/NprUIqJPWwZiZma2uJrVpyZpmKRfpuerSBrUumGZmVm5ZJfJKt+jmi00qUn6M/Bt4IA0aSZw\nSWsGZWZmZZRuPVOuRzVrzpD+LSNioKSxABExTVLHVo7LzMysxZqT1GZLakc2OARJywLzWjUqMzMr\nqyovsMqmOUntL8BNwPKSzgCGAme0alRmZlZW1d5sWC4LTWoRcZWkZ4Dt0qS9ImJ864ZlZmbWcs29\nTFZ7YDZZE6SvQmJmVkPqRj8WQXNGP54CXAusDKwC/EPSL1o7MDMzKx+Pfvyv4cCAiJgJIOksYCxw\ndmsGZmZm1lLNSWqT6y3XIU0zM7MaUd31Vfk0dUHjP5L1oU0DJki6J73eAXi6bcIzM7PFJRXn1jNN\nVWp1IxwnAHeUTH+i9cIxMzNbdE1d0PjytgzEzMxaT0EKtYX3qUlaAzgL6Ad0qpseEWu3YlxmZmYt\n1pxzzkYAV5L1M+4EjAKub8WYzMyszIoypL85SW2piLgHICJej4hfkSU3MzOrEVL5HtWsOUP6Z6UL\nGr8u6QhgEtC1dcMyMzNrueYkteOBpYGfkPWtLQMc3JpBmZlZ+Qh5SH+diHgyPZ3Of28UamZmtaIG\nmg3LpamTr28h3UOtIRGxZ6tEZGZmtoiaqtT+3GZR1KDOHdvTv88ylQ7DakyPTY+pdAhWY2a9/HZZ\ntlPtoxbLpamTrx9oy0DMzKz1FOWeYUU5TjMzK4Dm3iTUzMxqlHDz49dIWjIiZrVmMGZm1jp85+tE\n0maSXgBeTa83kvSnVo/MzMyshZrTp3YhsAvwEUBEPAd8uzWDMjOz8mqn8j2qWXOaH9tFxMR67bFz\nWykeMzMrs+yajVWejcqkOUntHUmbASGpPfBj4JXWDcvMzKzlmpPUjiRrglwV+AC4P00zM7MaUe3N\nhuXSnGs/fggMa4NYzMyslRSk9bFZd76+jAauARkRh7VKRGZmZouoOc2P95c87wR8D3indcIxM7Ny\nE/jWM3Ui4vrS15KuBh5ptYjMzKzsinJNxEU5ztWBFcsdiJmZ2eJqTp/ax/y3T60dMA04uTWDMjOz\n8ipI62PTSU3Z2XobAZPSpHkR0eiNQ83MrPpIKkyfWpPNjymB3RkRc9PDCc3MzKpWc/rUxkka0OqR\nmJlZq8kulVWeRzVrtPlRUoeImAMMAJ6W9DrwOdno0IiIgW0Uo5mZLSZfUQSeAgYCu7VRLGZmZoul\nqaQmgIh4vY1iMTOzVuCTrzPLSzqhsZkR8YdWiMfMzFpBQXJak0mtPdCFVLGZmZlVu6aS2uSI+E2b\nRWJmZq2jBu5YXS4L7VMzM7Pap4J8pTd1ntq2bRaFmZlZGTRaqUXEtLYMxMzMWkc2+rHSUbSN5txP\nzczMalxRklpRbrFjZmYF4ErNzKwAVJAT1VypmZnlXF2fWrkezdqn1F7SWEn/TK97SrpP0qvpb4+S\nZX8h6TVJL0vasWT6IEkvpHkXqhmZ2UnNzMxaw7HASyWvTwYeiIi1gAfSayT1A4YB6wNDgIsktU/r\nXAwcCqyVHkMWtlMnNTOzvCvjbWea04opaRVgZ+BvJZN3B0am5yOBPUqmXxcRsyLiTeA1YDNJvYBu\nEfFEupfnVSXrNMp9amZmBVDmCxovJ2lMyetLI+LSktfnAz8HupZMWzEiJqfn7wMrpue9gSdKlns3\nTZudntef3iQnNTMza6mpEbFJQzMk7QJ8GBHPSBrc0DIREZKiNQJzUjMzy7k2Pvl6K2A3Sd8FOgHd\nJF0DfCCpV0RMTk2LH6blJwF9StZfJU2blJ7Xn94k96mZmRVAW/WpRcQvImKViOhLNgDkXxGxPzAa\nODAtdiBwW3o+GhgmaUlJq5MNCHkqNVV+JmmLNOpxeMk6jXKlZmZmbeEcYJSkQ4CJwFCAiJggaRTw\nIjAHODoi5qZ1jgJGAJ2Bu9KjSU5qZma5J9pV4Cr9EfEg8GB6/hGNXCg/Is4Czmpg+higf0v26aRm\nZpZzojh3vnafmpmZ5YYrNTOzvPOdr83MLE/KfPJ11XLzo5mZ5YYrNTOznCvSQBEnNTOzAnDzo5mZ\nWY1xpWZmVgAFKdSc1MzM8k4Up1muKMdpZmYF4ErNzCzvBCpI+6OTmplZARQjpbn50czMcsSVmplZ\nzmV3vi5GreakZmZWAMVIaW5+NDOzHHGlZmZWAAVpfXRSMzPLPxVmSL+bH83MLDdcqZmZ5VyRLpPl\npGZmVgBufjQzM6sxrtTMzAqgGHWak5othsN/dDB33flPll9hBZ4ZNx6Am268gbP+93T+89JLPPzY\nUwzaZJMKR2mVcvQ+gzlozy2RxJU3P8qf//EgAEcO+xaHD/0mc+cFdz88nlMuuI0OHdpx8Wn7sfG6\nfejQvh1/v+MpzrviXrostST3X3H8/G32XqE71935ND8776bKHFSt8gWNzRbugAN/yBFHHcOPDh4+\nf9r66/fnulE3c8xRh1cwMqu0fmv04qA9t+SbB/yOr2bPZfRfjuLOh8ezyoo92GXwBmy29zl8NXsO\ny/foAsD3txvIkh07sOnQ39K50xKMvelXjLprDG9PnsYWw86Zv91H//5zbv3XuEodltUAJzVbZFt/\ncxsmvvXWAtPWXW+9ygRjVWXd1Vfi6fFv8cWXswF4+JnX2OM7GzOw36qcd+V9fDV7DgBTPp4BQBAs\n1akj7du3o/OSHflq9lymf/7lAttcc9UVWKFnVx599vW2PZgcKNLox6Icp5m1oQmvv8dWA9ak5zJL\n07nTEgzZen1WWakHa662AlsNWIOHrjqRe/92LIP6rQrAzfePZeaXX/HmfWfxyl2/4fyrHuDjz2Yu\nsM29hgzkxnufrcTh5IKksj2qmSs1Myu7l9/8gN+PuI/bLzqamV9+xXMvv8vcufPo0L4dPZdZmm2G\nn8cm66/GNecezHq7nM6m6/dl7tx5fGOHU+jRdSnuv+J4/vXkf3hr0kfzt7nXjoM45FdXVfCorBa0\nWaUm6bFFXG9jSSFpSMm07pKOKnndV9K+ixHbg5I8osGsjEbe+jhb7Xcu2x9yPp98NpNXJ37IpA8+\n4dYHsj6xMRMmMm9esFyPLgzdaRPufexF5syZx5SPZ/D4uDfmV3EAG6zdmw7t2zP2pXcqdTg1T2V8\nVLM2S2oRseUirroP8Ej6W6c7cFTJ677AIic1Myu/ukEgfVbqwe7f2Yjr7xrD7Q8+z7c2XRvI+sg6\nLtGBqR/P4N33pzF403UAWKpTRzbbsC8vv/XB/G0NHTKIUXePafuDsJrTZs2PkmZERBdJvYDrgW5p\n/0dGxMONrCNgL2B74GFJnSLiS+AcYA1J44D7gG8C66XXI4FbgKuBpdOmjomIx9I2TwL2B+YBd0XE\nySX7awdcAbwbEb9qIJ7DgMMA+qy6av3ZhTN8/314+N8PMnXqVNbouwqnnnYGPXr25ITjfszUKVPY\nc/ed2XADJCX1AAAR+0lEQVSjjbn9znsqHapVwLXn/Yie3Zdm9py5HHfOKD6d8QUjb32cv56+H2Nu\n+CVfzZ7Lj067GoBLrn+IS8/Yn2duPAUJrr7tCca/+t78bX1/+4Hs8eOLK3UouVDlXWFlo4homx39\nN6n9FOgUEWdJag8sFRHTG1lnK+A3EbGtpH8AN0XETZL6Av+MiP5pucHAiRGxS3q9FDAvIr6UtBZw\nbURsImkn4FRgu4iYKalnREyT9CBwMnAsMD4izlrY8QwatEk8+qR/OVrL9Nj0mEqHYDVm1sujmDfz\nw8VKSWutv1H84bp7yxUSu2240jMRUZVdNpUY/fg0cJCk04ENGktoyT7Aden5dSzYBNmUJYDLJL0A\n3AD0S9O3A66MiJkAETGtZJ2/0syEZmZm1anNk1pEPARsA0wCRkga3tByqYr7PnCapLeAPwFDJHVt\nxm6OBz4ANgI2ATo2Y53HgG9L6tSMZc3MaopUvkc1a/OkJmk14IOIuAz4GzCwkUW3BZ6PiD4R0Tci\nVgNuAr4HTAdKk1v918sAkyNiHnAA0D5Nv4+sSlwqxdKzZJ3LgTuBUZJ8qoOZ5YjK+l81q0Tz42Dg\nOUljgb2BCxpZbh+yAR+lbgL2iYiPgEcljZf0O+B5YK6k5yQdD1wEHCjpOWBd4HOAiLgbGA2MSYNK\nTizdeET8ARgLXJ0GjZiZWQ1ps4okIrqkvyPJRigubPmDGpg2miwpERH1h/B/p97rDUuen1SyjXPI\nRk+WbndwyfNfLyw2M7NaU+3NhuXiZjYzs5zLrv1YjKxWFUlN0pPAkvUmHxARL1QiHjMzq01VkdQi\nYvNKx2Bmlls1MGqxXKoiqZmZWesqSlLzCD8zM8sNV2pmZgVQ7eeXlYuTmplZzgloV4yc5uZHMzPL\nD1dqZmYF4OZHMzPLDY9+NDMzqzGu1MzMCsDNj2Zmlgse/WhmZlaDXKmZmeVe9d/cs1yc1MzM8q5A\nFzR286OZmeWGKzUzswIoSKHmpGZmlnfZ6MdipDU3P5qZWW64UjMzK4Bi1GlOamZmxVCQrObmRzMz\nyw1XamZmBeCTr83MLDcKMvjRzY9mZpYfrtTMzAqgIIWak5qZWSEUJKu5+dHMzHLDlZqZWc4Jj340\nM7O88K1nzMzMao+TmplZAaiMj4XuS+oj6f8kvShpgqRj0/Seku6T9Gr626NknV9Iek3Sy5J2LJk+\nSNILad6FUtM1p5OamVkRtGVWgznATyOiH7AFcLSkfsDJwAMRsRbwQHpNmjcMWB8YAlwkqX3a1sXA\nocBa6TGkqR07qZmZWVlFxOSIeDY9nw68BPQGdgdGpsVGAnuk57sD10XErIh4E3gN2ExSL6BbRDwR\nEQFcVbJOgzxQxMws91Sx0Y+S+gIDgCeBFSNicpr1PrBiet4beKJktXfTtNnpef3pjXJSMzMrgDKP\nflxO0piS15dGxKVf36e6ADcBx0XEZ6XdYRERkqKsUeGkZmZmLTc1IjZpagFJS5AltL9HxM1p8geS\nekXE5NS0+GGaPgnoU7L6KmnapPS8/vRGuU/NzCznyjlGpJmjHwVcDrwUEX8omTUaODA9PxC4rWT6\nMElLSlqdbEDIU6mp8jNJW6RtDi9Zp0Gu1MzMiqBtu9S2Ag4AXpA0Lk37JXAOMErSIcBEYChAREyQ\nNAp4kWzk5NERMTetdxQwAugM3JUejXJSMzOzsoqIR2g8jW7byDpnAWc1MH0M0L+5+3ZSMzMrAF/7\n0czMcsPXfjQzM6sxrtTMzAqgIIWak5qZWe41/5qNNc/Nj2Zmlhuu1MzMCsCjH83MLBeERz+amZnV\nHFdqZmYFUJBCzUnNzKwQCpLV3PxoZma54UrNzKwAPPrRzMxyw6MfzczMaowrNTOzAihIoeakZmZW\nCAXJam5+NDOz3HClZmaWc9lF+otRqjmpmZnlnTz60czMrOa4UjMzK4CCFGpOamZmhVCQrOaktoie\nffaZqZ2X0MRKx1GFlgOmVjoIqzn+3DRutUoHUEuc1BZRRCxf6RiqkaQxEbFJpeOw2uLPTWuTRz+a\nmVl+ePSjmZlZjXGlZuV2aaUDsJrkz00rEoUZJ+KkZuUVEf5yshbz56YNFCSrufnRzMxyw5WamVkB\nFGX0oys1MzPLDVdqVnGSegLLRcQrlY7Fao8kRURUOo5q5yH9Zm1AUifgJ8DBktardDxWOyT1AXBC\nax6V8VHNnNSsoiLiS+D+9HIvSf0qGY9VL0ldJHVMz9cDzpXUtcJhWZVxUrOKkbIGkYh4BBgNdAN+\n4MRm9UlaGvg7sFeaNDM9ZkhaIi1T7UVE5aT7qZXrUc2c1Kwi6vpBJK0uqUNEPAZcCSxDltjcFGnz\nRcTnwPXAQZL2BvoCX0RmdlrGzZBNKkYDpAeKWEWkhLYzcCrwsKQZwPlkV5Y4BNhf0t8j4sVKxmmV\nJ6l9RMyNiH9ImgKcBDwDrC7pAuBdYBbQISL+UMlYrfJcqVlFSNoC+C2wN9mPqz2Ac4EpwEhgaeCr\nigVoVSFV9HMlbS/p3Ii4D7gA2Jbs8/F2+tsFeLKCoVY1UZzmR1dq1qYktQOC7P5Zw4F1gW2Ak4HD\ngPPIfomfkpqcrMBSRb8tcBFweJp2u6Q5wAnAKxFxeyVjrBVVnovKxpWatYmSTvwuqR/knxHxHFmF\n9qOIuAf4kOyH1opOaKZMB2AIcGpE/Ktu9GNE3AVcApwkqXcl47Tq4qRmbaKkD+0BSadL2jPNWgE4\nTNLmwGbAeRExvmKBWtVIP37mAF8CW0jqFBFfAUjaFLgT2C0iJlUyzlpRlOZHJzVrE5J6AfuRNS9O\nA3ZMSe5goA9wGnB2RDxfuSit0uoqekmrSlolTb4LWAL4Vpq3EfBHYO2ImFaRQGuQyvhfNXOfmrU6\nSZsAGwGTIuJ6ScsDOwLfA5aIiF0kLRURM33Jo2IrqejPBh6T1DMihqZTPA6QdBLZaR9npuZrswU4\nqVmrkjSYbDTjPWTD9K+NiGcl3QV0BHaX9FREvAc+16ioSs5b3IJsFOwuZJXZFZLuj4jtJI0g+3H0\naUS87h9ALVTdBVbZOKlZq5G0OvBL4ICIeEjSa8A1kvaLiLGSbgPurktoVjzp2p+z07D9FYGPgKHA\nWmSjHZcBHpT0WERsCTxbt64TWssUJKe5T83Kq6RPZFOyX9rLkI1wJCLOBS4HRksaFBEfOaEVVzq9\nY0vgOEm7kPWrTgdeBHYGroiI6WSV/qrpM2XWJCc1K6vUhLQNWRPSC2QnWC8l6Zg0//fAX8hOljV7\nHtgBuBq4MSLeJysqJgNrSDqUrCly+4h4unJh1rZyjnz06EcrFEnrAEcCIyLiGeBB4AFgXUk/BYiI\ncyLi374AbTFJWlrSKhExD1gtTf4/YKc0bH8e2Z0bZpIltEsi4qUKhZsbRRn96KRm5bYBsCKwnaTl\nI+JT4G7gMWAdSXVfYu4TKa6+wJ8knQKcCPwU+DHZXRrqrt34Blmi+35E3OwfQNZcTmq2WEr60FaR\ntExE3Eh2keLPyK62v2zqF7kdOC0iJlYwXKsCETEBeI1sENGT6WT7KWSXwlpS0gNkFf7sdPK1fwCV\nQzEu0u/Rj7boJLWLiHmSdiLrQ3tZ0gpkA0P+CexEdm7R1RHxEdkgACsgSd2BryJiZpo0Hvg9MFzS\nCxHxAPB8qt62B96LiCcqFG4uVXkuKhsnNWsxSZ0j4ouU0NYE/hc4PCIek3QhcCvZydVLpL9Lkw3V\ntgKS1BN4Bbhf0sMR8ZeIGJnmvQP8QdKBwCfAnnW3j/F5aLYonNSsRSQtA5wj6ZaIuJfsi+g/ZF9a\nRMRPJF0LnBwRv5b0dERMrmDIVnkfA/eSjWjcT9JmwCPADRFxmaSvgJuAOcBxdSs5oZVXUXol3adm\nLdWNrD9k33RLkM+AZYHtSpa5k3QvNCc0S8npWbIBRNsAI9Lff0v6NtmAkM3JBoXcVak4862cYx+r\nOzu6UrNmkdQ1IqZHxDuSrgKGkV2MeApZh/8ISesCn6bpP69ctFZtIuI8SXeS/fgZD2xMVuEPA9YE\n9vbdGawcnNRsoST1BW6U9AwwCngVuBKYRTYc+/8Be5ENDFkZOD4i7nefiAFIah8Rc8kqtO+RXWH/\n8pToViC7qPXUSsaYd3V3vi4CJzVrjk5AL2B34C2yK4JcAvQgO//sVOCsiLigdCUnNANICQ3gSeB0\n4PGIOC9Nm+LPiZWT+9SsSWnY/n/Imo0+Bd4G9gbeI7u24w/S63MldU/X8zNbQKraJwInAF3q7lbt\nhGbl5krNmpSG7beLiJck7Q9cB/w2Ii6XdCPZ1dR3B8ZFxCcVDdYqquT2Me3Spa7mK0le7wLzvr62\ntTY3P5olJYntaUnDgGvTNfr+ArxMduK1zysqsJKEti1ZJXZPRHxZf7mIGC/ppIiYVIEwC63aRy2W\ni5uKrFlKExtZc+Opko6ut4wTWgGlgSAhaQhwMfBxQwlNmXYRMVHSUpKWbftoLe+c1GwBJddy/Npn\noySxPQPsCkxo6/isekhaM53qMVdSD7IBQ0ekG8J+U9KB6UTrOnWXVetOdm5az4oEXkQFuvWMmx9t\nvuY0IdWr2NzkWGwrAitIeiIiPpb0f8Ah6R5o7YDZZH2uT0nqEBFz0hVpbgB+FhGvVi70YqmB6xCX\njSs1A5rfhFS3eFqnM9mwfiugiHiU7Eawb0jqRnYe2lPAnyJib7JzGteX1DEltB7ALcBvIuKhSsVt\n+eakVnAtbUKqO5E2NSE9SHaJLCuodFuhY8nOV5waERekC1t/k+xC13+LiK/S4vsAZ0bEwxUKt9h8\n6xkrCDch2WKJiNskzQaekTQI+JLs/MVfRcQddU3UEXFRZSMttqKMfnRSK7iIeFRSV7ImpA3JmpB2\nBp5Ov7h3Aw5KTUhfpWruJuDX/sVtdSLiTknzgJeAdYCTIuLLkn5a971am3Dzo7kJycoiIu4GfgQM\nqOuPrUtkTmiV59GPVihuQrJyiIg7wKNiq1GV56KycVKz+dyEZOXiz4lVipsfbQFuQjLLKY9+tKJy\nE5JZ/hRl9KMrNWuUE5qZ1RpXamZmOVekO1/LP8bNzPJN0t3AcmXc5NSIGFLG7ZWNk5qZmeWG+9Qs\ntyTNlTRO0nhJN0haajG2NVjSP9Pz3SSd3MSy3SUdtQj7OF3Sic2dXm+ZEZJ+0IJ99ZU0vqUxmlU7\nJzXLsy8iYuOI6A98BRxROrPuppUt3WhEjI6Ic5pYpDvQ4qRmZovPSc2K4mFgzVShvCzpKmA80EfS\nDpIel/Rsqui6AEgaIuk/kp4F9qzbkKQfSvpzer6ipFskPZceWwLnAGukKvF3abmfSXpa0vOSzijZ\n1imSXpH0CNkJ702SdGjaznOSbqpXfW4naUza3i5p+faSfley78MX9400q2ZOapZ7kjoAO5Hd+wuy\nuw5cFBHrA58DvwK2i4iBwBjgBEmdgMvI7vA9CFipkc1fCPw7IjYCBpLdDfxk4PVUJf5M0g5pn5sB\nGwODJG2TLkc2LE37LrBpMw7n5ojYNO3vJeCQknl90z52Bi5Jx3AI8GlEbJq2f6ik1ZuxH7Oa5CH9\nlmedJY1Lzx8GLgdWBiZGxBNp+hZAP+BRZWOeOwKPA+sCb9bdWkfSNcBhDezjO8BwgIiYC3ya7mRQ\naof0GJtedyFLcl2BWyJiZtrH6GYcU39JZ5I1cXYB7imZNyoi5gGvSnojHcMOwIYl/W3LpH2/0ox9\nmdUcJzXLsy8iYuPSCSlxfV46CbgvIvapt9wC6y0mAWdHxF/r7eO4RdjWCGCPiHhO0g+BwSXz6g9l\njrTvH0dEafJDUt9F2LdZ1XPzoxXdE8BWktYEkLS0pLWB/wB9Ja2RltunkfUfAI5M67ZPN1CdTlaF\n1bkHOLikr663pBWAh4A9JHVO97TbtRnxdgUmS1oC2K/evL0ktUsxfwN4Oe37yLQ8ktaWtHQz9mNW\nk1ypWaFFxJRU8Vwrack0+VcR8Yqkw4A7JM0ka77s2sAmjgUulXQIMBc4MiIel/RoGjJ/V+pXWw94\nPFWKM4D9I+JZSdcDzwEfAk83I+RTgSeBKelvaUxvA08B3YAj0h0W/kbW1/assp1PAfZo3rtjVnt8\n8rWZmeWGmx/NzCw3nNTMzCw3nNTMzCw3nNTMzCw3nNTMzCw3nNTMzCw3nNTMzCw3nNTMzCw3/j/U\nBJUL3Q/H1AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f4bbe2f9ac8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot(actual_value = Train.actual_value_, pred_value = Train.pred_value_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-14T18:59:23.676878Z",
     "start_time": "2017-06-14T18:59:23.632251Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#4.5 GB\n",
    "pd.Series(Train.pred_value).to_csv('LSTM_prediction_values.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "_draft": {
   "nbviewer_url": "https://gist.github.com/7d1ace18a82178e15ece8fc5252fce88"
  },
  "anaconda-cloud": {},
  "gist": {
   "data": {
    "description": "Hyper parameter tuning",
    "public": false
   },
   "id": "7d1ace18a82178e15ece8fc5252fce88"
  },
  "kernelspec": {
   "display_name": "Python [conda env:p3]",
   "language": "python",
   "name": "conda-env-p3-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
